From ddfd192b704dddfefe2dd78345de239995a40159 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sat, 11 Jun 2022 23:51:34 +0430 Subject: [PATCH 1/4] add tests for multiple headers with same name --- tests/test_http_headers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 64ff7a73d..0c51fd701 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -38,6 +38,13 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2']) assert h.getlist('X-Forwarded-For') is not hlist + def test_multivalue_for_one_header(self): + h = Headers((("a", "b"), ("a", "c"))) + self.assertEqual(h["a"], b"c") + self.assertEqual(h.get("a"), b"c") + self.assertEqual(h.getlist("a"), [b"b", b"c"]) + assert h.getlist("a") is not ["b", "c"] + def test_encode_utf8(self): h = Headers({'key': '\xa3'}, encoding='utf-8') key, val = dict(h).popitem() From 6a0bcf97cc6016cb966b92170709bc7518cf62c2 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sat, 11 Jun 2022 23:52:21 +0430 Subject: [PATCH 2/4] Merge values of multiple headers with same name (#5515) --- scrapy/http/headers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 1a2b99b0a..a9471d721 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,6 +1,7 @@ from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaselessDict from scrapy.utils.python import to_unicode +from collections.abc import Mapping class Headers(CaselessDict): @@ -10,6 +11,13 @@ class Headers(CaselessDict): self.encoding = encoding super().__init__(seq) + def update(self, seq): + seq = seq.items() if isinstance(seq, Mapping) else seq + iseq = {} + for k, v in seq: + iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) + super().update(iseq) + def normkey(self, key): """Normalize key to bytes""" return self._tobytes(key.title()) @@ -86,4 +94,5 @@ class Headers(CaselessDict): def __copy__(self): return self.__class__(self) + copy = __copy__ From a135d6caf050f4b7b5af28d4cccc5d5ef51dbaf6 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Mon, 13 Jun 2022 15:24:30 +0430 Subject: [PATCH 3/4] Move Mapping import line up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/http/headers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index a9471d721..9c03fe54f 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,7 +1,8 @@ +from collections.abc import Mapping + from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaselessDict from scrapy.utils.python import to_unicode -from collections.abc import Mapping class Headers(CaselessDict): From 892c2a46554bdf80d49d3f28cc012c49cd1e19ca Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Mon, 13 Jun 2022 23:46:42 +0430 Subject: [PATCH 4/4] delete unnecessary test --- tests/test_http_headers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 0c51fd701..1ca936247 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -43,7 +43,6 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h["a"], b"c") self.assertEqual(h.get("a"), b"c") self.assertEqual(h.getlist("a"), [b"b", b"c"]) - assert h.getlist("a") is not ["b", "c"] def test_encode_utf8(self): h = Headers({'key': '\xa3'}, encoding='utf-8')