From ac111088c6470d55e4d698247a76a4938f4a6dc6 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 20:12:29 +0530
Subject: [PATCH 01/28] duplicate keys handled

---
 scrapy/http/request/form.py | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index c2413b431..660f4fc96 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst
 """
 
 import six
-from six.moves.urllib.parse import urljoin, urlencode
+from six.moves.urllib.parse import urljoin, urlencode, urlsplit
 
 import lxml.html
 from parsel.selector import create_root_node
@@ -33,7 +33,30 @@ class FormRequest(Request):
                 self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
                 self._set_body(querystr)
             else:
-                self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr)
+                if urlsplit(self.url).query:
+                    queries = (urlsplit(self.url).query + '&' + querystr).split('&')
+                else:
+                    queries = querystr.split('&')
+                query_dict = {}
+                for i in range(len(queries)):
+                    query_list = queries[i].split('=')
+                    query_dict[query_list[0]] = query_list[1]
+                querystr = ''
+                query_key = list(query_dict.keys())
+                for i in range(len(query_dict)):
+                    querystr += (query_key[i] + '=' + query_dict[query_key[i]])
+                    if i!=len(query_dict)-1:
+                        querystr += '&'
+                if urlsplit(self.url).fragment:
+                    if urlsplit(self.url).query:
+                        self._set_url(self.url[:self.url.index('?')+1]+querystr+'#'+urlsplit(self.url).fragment)
+                    else:
+                        self._set_url(self.url+'?'+querystr+'#'+urlsplit(self.url).fragment)
+                else:
+                    if urlsplit(self.url).query:
+                        self._set_url(self.url[:self.url.index('?')+1]+querystr)
+                    else:
+                        self._set_url(self.url+'?'+querystr)
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From b5e454809e240839e76f050b4587abcd80c33765 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 20:12:31 +0530
Subject: [PATCH 02/28] Included test

---
 tests/test_http_request.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 58326a384..5a7cf3918 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -273,6 +273,19 @@ class FormRequestTest(RequestTest):
     def test_empty_formdata(self):
         r1 = self.request_class("http://www.example.com", formdata={})
         self.assertEqual(r1.body, b'')
+        
+    def test_formdata_overrides_querystring_duplicates(self):
+        data1 = {'a' : '1', 'b' : '2'}
+        data2 = [('a','one'), ('a','two'), ('c','three')]
+    
+        fs1 = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1))
+        self.assertEqual(fs1[b'a'], [b'1'])
+        self.assertEqual(fs1[b'b'], [b'2'])
+
+        fs2 = _qs(self.request_class('http://www.example.com?a=1&b=2&c=3', method='GET', formdata=data2))
+        self.assertEqual(fs2[b'a'], [b'two'])
+        self.assertEqual(fs2[b'b'], [b'2'])
+        self.assertEqual(fs2[b'c'], [b'three'])
 
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)

From 1bea5d307628a11d4010cdabb87b2a29a6c3c772 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 22:35:16 +0530
Subject: [PATCH 03/28] Fixed error

---
 scrapy/http/request/form.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 660f4fc96..f08d1f626 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -34,29 +34,32 @@ class FormRequest(Request):
                 self._set_body(querystr)
             else:
                 if urlsplit(self.url).query:
-                    queries = (urlsplit(self.url).query + '&' + querystr).split('&')
+                    queries = (urlsplit(self.url).query).split('&')
                 else:
                     queries = querystr.split('&')
                 query_dict = {}
+                duplicate_key_passed=[]
+                for i in range(len(querystr.split('&'))):
+                    query_list = querystr.split('&')[i].split('=')
+                    duplicate_key_passed.append(query_list[0])
                 for i in range(len(queries)):
                     query_list = queries[i].split('=')
-                    query_dict[query_list[0]] = query_list[1]
-                querystr = ''
+                    if duplicate_key_passed.count(query_list[0])<=1:
+                        query_dict[query_list[0]] = query_list[1]  
+                query_str = ''
                 query_key = list(query_dict.keys())
                 for i in range(len(query_dict)):
-                    querystr += (query_key[i] + '=' + query_dict[query_key[i]])
-                    if i!=len(query_dict)-1:
-                        querystr += '&'
+                    query_str += (query_key[i] + '=' + query_dict[query_key[i]] + '&')       
                 if urlsplit(self.url).fragment:
                     if urlsplit(self.url).query:
-                        self._set_url(self.url[:self.url.index('?')+1]+querystr+'#'+urlsplit(self.url).fragment)
+                        self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + '#' + urlsplit(self.url).fragment)
                     else:
-                        self._set_url(self.url+'?'+querystr+'#'+urlsplit(self.url).fragment)
+                        self._set_url(self.url + '?' + querystr + '#' + urlsplit(self.url).fragment)
                 else:
                     if urlsplit(self.url).query:
-                        self._set_url(self.url[:self.url.index('?')+1]+querystr)
+                        self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr)
                     else:
-                        self._set_url(self.url+'?'+querystr)
+                        self._set_url(self.url + '?' + querystr)
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From 9a4bbd6d029f61dc60594fb8bca631c3a494f855 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 23:05:58 +0530
Subject: [PATCH 04/28] Update form.py

---
 scrapy/http/request/form.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index f08d1f626..1b487cbd0 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -44,7 +44,7 @@ class FormRequest(Request):
                     duplicate_key_passed.append(query_list[0])
                 for i in range(len(queries)):
                     query_list = queries[i].split('=')
-                    if duplicate_key_passed.count(query_list[0])<=1:
+                    if duplicate_key_passed.count(query_list[0])==0:
                         query_dict[query_list[0]] = query_list[1]  
                 query_str = ''
                 query_key = list(query_dict.keys())

From 4abcdcb306d8782e6b792ad36e40391a0bcbd0f3 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 23:22:53 +0530
Subject: [PATCH 05/28] Update test_http_request.py

---
 tests/test_http_request.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 5a7cf3918..ec247a02d 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -276,17 +276,10 @@ class FormRequestTest(RequestTest):
         
     def test_formdata_overrides_querystring_duplicates(self):
         data1 = {'a' : '1', 'b' : '2'}
-        data2 = [('a','one'), ('a','two'), ('c','three')]
-    
-        fs1 = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1))
+        fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1))
         self.assertEqual(fs1[b'a'], [b'1'])
         self.assertEqual(fs1[b'b'], [b'2'])
 
-        fs2 = _qs(self.request_class('http://www.example.com?a=1&b=2&c=3', method='GET', formdata=data2))
-        self.assertEqual(fs2[b'a'], [b'two'])
-        self.assertEqual(fs2[b'b'], [b'2'])
-        self.assertEqual(fs2[b'c'], [b'three'])
-
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)
         data = {b'one': b'two', b'price': b'\xc2\xa3 100'}

From 023290dabc55c92ea2143a98f7753abaf21f74c9 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 23:50:31 +0530
Subject: [PATCH 06/28] Update test_http_request.py

---
 tests/test_http_request.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index ec247a02d..1ef84f1d6 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -275,10 +275,10 @@ class FormRequestTest(RequestTest):
         self.assertEqual(r1.body, b'')
         
     def test_formdata_overrides_querystring_duplicates(self):
-        data1 = {'a' : '1', 'b' : '2'}
-        fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1))
-        self.assertEqual(fs1[b'a'], [b'1'])
-        self.assertEqual(fs1[b'b'], [b'2'])
+        data = {'a' : '1', 'b' : '2'}
+        fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data))
+        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(fs[b'b'], [b'2'])
 
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)

From 9f1f4df9666f38f81adda5a41f3cac646a4f18dc Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Wed, 16 Jan 2019 22:59:41 +0530
Subject: [PATCH 07/28] Update test_http_request.py

---
 tests/test_http_request.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 1ef84f1d6..6c4cb10a8 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -275,8 +275,22 @@ class FormRequestTest(RequestTest):
         self.assertEqual(r1.body, b'')
         
     def test_formdata_overrides_querystring_duplicates(self):
+        #Without fragment
         data = {'a' : '1', 'b' : '2'}
-        fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data))
+        fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data))
+        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(fs[b'b'], [b'2'])
+
+        #With fragment
+        data = (('a', '1'), ('b', '2'))
+        url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0]
+        fs = _qs(self.request_class(url, method='GET', formdata=data))
+        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(fs[b'b'], [b'2'])
+
+        #Witout query
+        data = {'a' : '1', 'b' : '2'}
+        fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
 

From 3e67fa8fc1eaa2189cec08ce4000b3efc254ee1b Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Wed, 16 Jan 2019 23:01:38 +0530
Subject: [PATCH 08/28] Improved for better user readability

---
 scrapy/http/request/form.py | 45 ++++++++++++++++---------------------
 1 file changed, 19 insertions(+), 26 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 1b487cbd0..fa5110382 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -33,33 +33,26 @@ class FormRequest(Request):
                 self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
                 self._set_body(querystr)
             else:
-                if urlsplit(self.url).query:
-                    queries = (urlsplit(self.url).query).split('&')
+                url_split = urlsplit(self.url)
+                formdata_key_list = []
+                for k in querystr.split('&'):
+                    formdata_key_list.append(k.split('=')[0])
+
+                if url_split.query:
+                    queries = (url_split.query).split('&')
+                    query_dict = {}
+
+                    for x in queries:
+                        query = x.split('=')
+                        if formdata_key_list.count(query[0])==0:
+                            query_dict[query[0]] = query[1]
+                        query_str = ''
+                        for k, v in query_dict:
+                            query_str += (k + '=' + v + '&')
+
+                    self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else ''))
                 else:
-                    queries = querystr.split('&')
-                query_dict = {}
-                duplicate_key_passed=[]
-                for i in range(len(querystr.split('&'))):
-                    query_list = querystr.split('&')[i].split('=')
-                    duplicate_key_passed.append(query_list[0])
-                for i in range(len(queries)):
-                    query_list = queries[i].split('=')
-                    if duplicate_key_passed.count(query_list[0])==0:
-                        query_dict[query_list[0]] = query_list[1]  
-                query_str = ''
-                query_key = list(query_dict.keys())
-                for i in range(len(query_dict)):
-                    query_str += (query_key[i] + '=' + query_dict[query_key[i]] + '&')       
-                if urlsplit(self.url).fragment:
-                    if urlsplit(self.url).query:
-                        self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + '#' + urlsplit(self.url).fragment)
-                    else:
-                        self._set_url(self.url + '?' + querystr + '#' + urlsplit(self.url).fragment)
-                else:
-                    if urlsplit(self.url).query:
-                        self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr)
-                    else:
-                        self._set_url(self.url + '?' + querystr)
+                    self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else ''))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From 6f86c93f366509ba05faf3b2cdc12f5e3ef6693a Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Wed, 16 Jan 2019 23:54:35 +0530
Subject: [PATCH 09/28] Increased test cases

---
 tests/test_http_request.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 6c4cb10a8..571df0176 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -275,20 +275,27 @@ class FormRequestTest(RequestTest):
         self.assertEqual(r1.body, b'')
         
     def test_formdata_overrides_querystring_duplicates(self):
-        #Without fragment
-        data = {'a' : '1', 'b' : '2'}
-        fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data))
-        self.assertEqual(fs[b'a'], [b'1'])
-        self.assertEqual(fs[b'b'], [b'2'])
-
-        #With fragment
+        #Both fragment and query in url
         data = (('a', '1'), ('b', '2'))
         url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
 
-        #Witout query
+        #only query in url
+        data = {'a' : '1', 'b' : '2'}
+        fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data))
+        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(fs[b'b'], [b'2'])
+
+        #only fragment in url
+        data = (('a', '1'), ('b', '2'))
+        url = self.request_class('http://www.example.com/#fragment', method='GET', formdata=data).url.split('#')[0]
+        fs = _qs(self.request_class(url, method='GET', formdata=data))
+        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(fs[b'b'], [b'2'])
+
+        #None of both in url
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])

From 6be73f06c33a27cb3291a1e46d80574b4d23b10b Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Thu, 17 Jan 2019 23:50:58 +0530
Subject: [PATCH 10/28] Updated tests

---
 tests/test_http_request.py | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 571df0176..a17acf93b 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -277,23 +277,11 @@ class FormRequestTest(RequestTest):
     def test_formdata_overrides_querystring_duplicates(self):
         #Both fragment and query in url
         data = (('a', '1'), ('b', '2'))
-        url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0]
-        fs = _qs(self.request_class(url, method='GET', formdata=data))
-        self.assertEqual(fs[b'a'], [b'1'])
-        self.assertEqual(fs[b'b'], [b'2'])
-
-        #only query in url
-        data = {'a' : '1', 'b' : '2'}
-        fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data))
-        self.assertEqual(fs[b'a'], [b'1'])
-        self.assertEqual(fs[b'b'], [b'2'])
-
-        #only fragment in url
-        data = (('a', '1'), ('b', '2'))
-        url = self.request_class('http://www.example.com/#fragment', method='GET', formdata=data).url.split('#')[0]
+        url = self.request_class('http://www.example.com/?a=0&a=2&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
+        self.assertEqual(fs[b'c'], [b'3'])
 
         #None of both in url
         data = {'a' : '1', 'b' : '2'}

From a9f68acb6dd7f5aca2bee95353278e5a40c32b8b Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Thu, 17 Jan 2019 23:51:09 +0530
Subject: [PATCH 11/28] modified code

---
 scrapy/http/request/form.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index fa5110382..2305491e2 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -41,15 +41,14 @@ class FormRequest(Request):
                 if url_split.query:
                     queries = (url_split.query).split('&')
                     query_dict = {}
-
                     for x in queries:
-                        query = x.split('=')
-                        if formdata_key_list.count(query[0])==0:
-                            query_dict[query[0]] = query[1]
+                        k = x.split('=')[0]
+                        v = x.split('=')[1]
+                        if formdata_key_list.count(k)==0:
+                            query_dict[k] = v
                         query_str = ''
-                        for k, v in query_dict:
+                        for k, v in query_dict.items():
                             query_str += (k + '=' + v + '&')
-
                     self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else ''))
                 else:
                     self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else ''))

From 7dee841b8b904a3961481f7b1999d819d2c17c5d Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sat, 19 Jan 2019 13:20:01 +0530
Subject: [PATCH 12/28] Update form.py

---
 scrapy/http/request/form.py | 21 +++++----------------
 1 file changed, 5 insertions(+), 16 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 2305491e2..af4ed1793 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst
 """
 
 import six
-from six.moves.urllib.parse import urljoin, urlencode, urlsplit
+from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl
 
 import lxml.html
 from parsel.selector import create_root_node
@@ -37,21 +37,10 @@ class FormRequest(Request):
                 formdata_key_list = []
                 for k in querystr.split('&'):
                     formdata_key_list.append(k.split('=')[0])
-
-                if url_split.query:
-                    queries = (url_split.query).split('&')
-                    query_dict = {}
-                    for x in queries:
-                        k = x.split('=')[0]
-                        v = x.split('=')[1]
-                        if formdata_key_list.count(k)==0:
-                            query_dict[k] = v
-                        query_str = ''
-                        for k, v in query_dict.items():
-                            query_str += (k + '=' + v + '&')
-                    self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else ''))
-                else:
-                    self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else ''))
+                items = []
+                items += [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list]
+                query_str = _urlencode(items, self.encoding)
+                self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else '')))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From 722a30ac2bc35bdc7abb52d10417ed1191a3aee0 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sat, 19 Jan 2019 13:20:05 +0530
Subject: [PATCH 13/28] Update test_http_request.py

---
 tests/test_http_request.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index a17acf93b..955db9e76 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -276,10 +276,10 @@ class FormRequestTest(RequestTest):
         
     def test_formdata_overrides_querystring_duplicates(self):
         #Both fragment and query in url
-        data = (('a', '1'), ('b', '2'))
-        url = self.request_class('http://www.example.com/?a=0&a=2&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
+        data = (('a', 'one'), ('a', 'two'), ('b', '2'))
+        url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
-        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
         self.assertEqual(fs[b'c'], [b'3'])
 

From 6eca6f92c6ae48bb136311308e77e82d7659cf43 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 4 Mar 2019 14:59:34 +0530
Subject: [PATCH 14/28] Update form.py

---
 scrapy/http/request/form.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index af4ed1793..b6ca6ef9c 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -34,11 +34,8 @@ class FormRequest(Request):
                 self._set_body(querystr)
             else:
                 url_split = urlsplit(self.url)
-                formdata_key_list = []
-                for k in querystr.split('&'):
-                    formdata_key_list.append(k.split('=')[0])
-                items = []
-                items += [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list]
+                formdata_key_list = list(dict(parse_qsl(querystr)).keys())
+                items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list]
                 query_str = _urlencode(items, self.encoding)
                 self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else '')))
 

From d75b61b96aad172e21c7d3aeba6f48419a63c72d Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 4 Mar 2019 15:07:12 +0530
Subject: [PATCH 15/28] Update test_http_request.py

---
 tests/test_http_request.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 955db9e76..2da1cdf6d 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -289,6 +289,12 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
 
+        #GET duplicates are preserved
+        formdata=(('foo', 'bar'), ('foo', 'baz'))
+        fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data))
+        self.assertEqual(set(fs[b'foo']), {b'bar', b'baz'})
+        self.assertEqual(set(fs[b'a']), {b'1', b'2'})
+
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)
         data = {b'one': b'two', b'price': b'\xc2\xa3 100'}

From fdf03a6d0daae79b9a3b421d19102873d5a75f46 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 4 Mar 2019 15:12:44 +0530
Subject: [PATCH 16/28] correcting tests

---
 tests/test_http_request.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 2da1cdf6d..0c2f95262 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -289,10 +289,10 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
 
-        #GET duplicates are preserved
-        formdata=(('foo', 'bar'), ('foo', 'baz'))
+        #Duplicate GET arguments are preserved
+        formdata={'foo' : 'bar'}
         fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data))
-        self.assertEqual(set(fs[b'foo']), {b'bar', b'baz'})
+        self.assertEqual(fs[b'foo'], [b'bar'])
         self.assertEqual(set(fs[b'a']), {b'1', b'2'})
 
     def test_default_encoding_bytes(self):

From 8831fafabc699bd2ab48e3e185712fbcd4f7cc7f Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 4 Mar 2019 15:42:48 +0530
Subject: [PATCH 17/28] Update test_http_request.py

---
 tests/test_http_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 0c2f95262..5c30c9785 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -290,7 +290,7 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[b'b'], [b'2'])
 
         #Duplicate GET arguments are preserved
-        formdata={'foo' : 'bar'}
+        data={'foo' : 'bar'}
         fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data))
         self.assertEqual(fs[b'foo'], [b'bar'])
         self.assertEqual(set(fs[b'a']), {b'1', b'2'})

From 7da460b7935940c93d5454019ea99f9f117b3e7b Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 4 Mar 2019 17:25:15 +0530
Subject: [PATCH 18/28] Update form.py

---
 scrapy/http/request/form.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index b6ca6ef9c..b44d7b9a6 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst
 """
 
 import six
-from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl
+from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl, urlunsplit
 
 import lxml.html
 from parsel.selector import create_root_node
@@ -34,10 +34,10 @@ class FormRequest(Request):
                 self._set_body(querystr)
             else:
                 url_split = urlsplit(self.url)
-                formdata_key_list = list(dict(parse_qsl(querystr)).keys())
-                items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list]
+                formdata_keys = set(dict(parse_qsl(querystr)).keys())
+                items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys]
                 query_str = _urlencode(items, self.encoding)
-                self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else '')))
+                self._set_url(urlunsplit((url_split.scheme, url_split.netloc, url_split.path, (query_str + '&' + querystr) if query_str else querystr, url_split.fragment)))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From f7bf3abbd03a10124ed648d45abbb6c8eb3218b7 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Wed, 6 Mar 2019 14:10:03 +0530
Subject: [PATCH 19/28] Modified code

---
 scrapy/http/request/form.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index b44d7b9a6..a0e7f76a9 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -37,7 +37,8 @@ class FormRequest(Request):
                 formdata_keys = set(dict(parse_qsl(querystr)).keys())
                 items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys]
                 query_str = _urlencode(items, self.encoding)
-                self._set_url(urlunsplit((url_split.scheme, url_split.netloc, url_split.path, (query_str + '&' + querystr) if query_str else querystr, url_split.fragment)))
+                query = (query_str + '&' + querystr) if query_str else querystr
+                self._set_url(urlunsplit(url_split._replace(query = query)))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From 35f7595dbebe7012c8f66336b76625b089a1fad3 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 11 Mar 2019 23:58:37 +0530
Subject: [PATCH 20/28] changed variable names

---
 scrapy/http/request/form.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index a0e7f76a9..9af2db5ff 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -28,16 +28,16 @@ class FormRequest(Request):
 
         if formdata:
             items = formdata.items() if isinstance(formdata, dict) else formdata
-            querystr = _urlencode(items, self.encoding)
+            form_query_str = _urlencode(items, self.encoding)
             if self.method == 'POST':
                 self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
-                self._set_body(querystr)
+                self._set_body(form_query_str)
             else:
                 url_split = urlsplit(self.url)
-                formdata_keys = set(dict(parse_qsl(querystr)).keys())
+                formdata_keys = set(dict(parse_qsl(form_query_str)).keys())
                 items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys]
-                query_str = _urlencode(items, self.encoding)
-                query = (query_str + '&' + querystr) if query_str else querystr
+                url_query_str = _urlencode(items, self.encoding)
+                query = (url_query_str + '&' + form_query_str) if url_query_str else form_query_str
                 self._set_url(urlunsplit(url_split._replace(query = query)))
 
     @classmethod

From 282f24c510bdc6fff0e86ca16b7d681617af25ca Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Wed, 20 Mar 2019 18:46:22 +0530
Subject: [PATCH 21/28] Update form.py

---
 scrapy/http/request/form.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 9af2db5ff..03692f063 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst
 """
 
 import six
-from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl, urlunsplit
+from six.moves.urllib.parse import urljoin, urlencode, urlsplit, urlunsplit
 
 import lxml.html
 from parsel.selector import create_root_node
@@ -33,12 +33,7 @@ class FormRequest(Request):
                 self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
                 self._set_body(form_query_str)
             else:
-                url_split = urlsplit(self.url)
-                formdata_keys = set(dict(parse_qsl(form_query_str)).keys())
-                items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys]
-                url_query_str = _urlencode(items, self.encoding)
-                query = (url_query_str + '&' + form_query_str) if url_query_str else form_query_str
-                self._set_url(urlunsplit(url_split._replace(query = query)))
+                self._set_url(urlunsplit(urlsplit(self.url)._replace(query = form_query_str)))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From 4c89e53e684987c61857ef810374101440d5616c Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Wed, 20 Mar 2019 18:46:25 +0530
Subject: [PATCH 22/28] Update test_http_request.py

---
 tests/test_http_request.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 5c30c9785..feea54a54 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -275,26 +275,17 @@ class FormRequestTest(RequestTest):
         self.assertEqual(r1.body, b'')
         
     def test_formdata_overrides_querystring_duplicates(self):
-        #Both fragment and query in url
         data = (('a', 'one'), ('a', 'two'), ('b', '2'))
         url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
-        self.assertEqual(fs[b'c'], [b'3'])
 
-        #None of both in url
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
 
-        #Duplicate GET arguments are preserved
-        data={'foo' : 'bar'}
-        fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data))
-        self.assertEqual(fs[b'foo'], [b'bar'])
-        self.assertEqual(set(fs[b'a']), {b'1', b'2'})
-
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)
         data = {b'one': b'two', b'price': b'\xc2\xa3 100'}

From dc8310e2929d228a224f12055a26cfef71d751cd Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Tue, 26 Mar 2019 15:42:58 +0530
Subject: [PATCH 23/28] changed tests

---
 tests/test_http_request.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index feea54a54..81c1a4a9e 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -274,12 +274,13 @@ class FormRequestTest(RequestTest):
         r1 = self.request_class("http://www.example.com", formdata={})
         self.assertEqual(r1.body, b'')
         
-    def test_formdata_overrides_querystring_duplicates(self):
+    def test_formdata_overrides_querystring(self):
         data = (('a', 'one'), ('a', 'two'), ('b', '2'))
         url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
+        self.assertNone(fs[b'c'])
 
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))

From 213b9eb879c4c6d8ae60ffa950688a38951b3349 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Tue, 26 Mar 2019 15:59:38 +0530
Subject: [PATCH 24/28] Update test_http_request.py

---
 tests/test_http_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 81c1a4a9e..56c7d3d94 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -280,7 +280,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
-        self.assertNone(fs[b'c'])
+        self.assertIsNone(fs[b'c'])
 
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))

From ae856e8ba835d1d99510e9509a462b55a757e85f Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Tue, 26 Mar 2019 16:21:52 +0530
Subject: [PATCH 25/28] corrected tests

---
 tests/test_http_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 56c7d3d94..55fa4ad22 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -280,7 +280,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
-        self.assertIsNone(fs[b'c'])
+        self.assertIsNone(fs.get([b'c']))
 
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))

From 5f2ad5377e54ecdb3059891db4887eb09e56bfc6 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Tue, 26 Mar 2019 16:46:15 +0530
Subject: [PATCH 26/28] fixed typo

---
 tests/test_http_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 55fa4ad22..89e4a8683 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -280,7 +280,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
-        self.assertIsNone(fs.get([b'c']))
+        self.assertIsNone(fs.get(b'c'))
 
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))

From 1d77eac950966463faee411b6196f197a8d32d4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Apr 2020 14:57:55 +0200
Subject: [PATCH 27/28] Fix Flake8-reported issues

---
 scrapy/http/request/form.py | 2 +-
 tests/test_http_request.py  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index d58d217b6..bdb6bec7a 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -33,7 +33,7 @@ class FormRequest(Request):
                 self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
                 self._set_body(form_query_str)
             else:
-                self._set_url(urlunsplit(urlsplit(self.url)._replace(query = form_query_str)))
+                self._set_url(urlunsplit(urlsplit(self.url)._replace(query=form_query_str)))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index bc34bb26d..96954d419 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -369,7 +369,7 @@ class FormRequestTest(RequestTest):
     def test_empty_formdata(self):
         r1 = self.request_class("http://www.example.com", formdata={})
         self.assertEqual(r1.body, b'')
-        
+
     def test_formdata_overrides_querystring(self):
         data = (('a', 'one'), ('a', 'two'), ('b', '2'))
         url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
@@ -378,7 +378,7 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[b'b'], [b'2'])
         self.assertIsNone(fs.get(b'c'))
 
-        data = {'a' : '1', 'b' : '2'}
+        data = {'a': '1', 'b': '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])

From 66e200423943a921326334a37777720c706ee2b5 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Mon, 14 Jun 2021 18:58:35 +0500
Subject: [PATCH 28/28] Fix a flake8 problem

---
 tests/test_http_request.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index ea32c6711..b610087bd 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -381,7 +381,8 @@ class FormRequestTest(RequestTest):
 
     def test_formdata_overrides_querystring(self):
         data = (('a', 'one'), ('a', 'two'), ('b', '2'))
-        url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
+        url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment',
+                                 method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])