diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 2540be01a..822597c84 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from w3lib.http import headers_dict_to_raw -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict from scrapy.utils.python import to_unicode @@ -88,7 +88,7 @@ class Headers(CaselessDict): """Return headers as a CaselessDict with unicode keys and unicode values. Multiple values are joined with ','. """ - return CaselessDict( + return CaseInsensitiveDict( ( to_unicode(key, encoding=self.encoding), to_unicode(b",".join(value), encoding=self.encoding), diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4b594ccb7..5c09ab37e 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -28,7 +28,7 @@ from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import md5sum @@ -155,7 +155,7 @@ class S3FilesStore: def _headers_to_botocore_kwargs(self, headers): """Convert headers to botocore keyword arguments.""" # This is required while we need to support both boto and botocore. - mapping = CaselessDict( + mapping = CaseInsensitiveDict( { "Content-Type": "ContentType", "Cache-Control": "CacheControl", diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index fa57a4f26..14b6e79ff 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -6,13 +6,29 @@ This module must not depend on any module outside the Standard Library. """ import collections +import warnings import weakref from collections.abc import Mapping +from typing import Any, AnyStr + +from scrapy.exceptions import ScrapyDeprecationWarning class CaselessDict(dict): __slots__ = () + def __new__(cls, *args, **kwargs): + from scrapy.http.headers import Headers + + if issubclass(cls, CaselessDict) and not issubclass(cls, Headers): + warnings.warn( + "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return super().__new__(cls, *args, **kwargs) + def __init__(self, seq=None): super().__init__() if seq: @@ -64,6 +80,48 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) +class CaseInsensitiveDict(collections.UserDict): + """A dict-like structure that accepts strings or bytes + as keys and allows case-insensitive lookups. + """ + + def __init__(self, *args, **kwargs) -> None: + self._keys: dict = {} + super().__init__(*args, **kwargs) + + def __getitem__(self, key: AnyStr) -> Any: + normalized_key = self._normkey(key) + return super().__getitem__(self._keys[normalized_key.lower()]) + + def __setitem__(self, key: AnyStr, value: Any) -> None: + normalized_key = self._normkey(key) + try: + lower_key = self._keys[normalized_key.lower()] + del self[lower_key] + except KeyError: + pass + super().__setitem__(normalized_key, self._normvalue(value)) + self._keys[normalized_key.lower()] = normalized_key + + def __delitem__(self, key: AnyStr) -> None: + normalized_key = self._normkey(key) + stored_key = self._keys.pop(normalized_key.lower()) + super().__delitem__(stored_key) + + def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + normalized_key = self._normkey(key) + return normalized_key.lower() in self._keys + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}: {super().__repr__()}>" + + def _normkey(self, key: AnyStr) -> AnyStr: + return key + + def _normvalue(self, value: Any) -> Any: + return value + + class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index b6a84ee91..9e5f88f48 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,9 +1,13 @@ import copy import unittest +import warnings from collections.abc import Mapping, MutableMapping +from typing import Iterator +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.utils.datatypes import ( + CaseInsensitiveDict, CaselessDict, LocalCache, LocalWeakReferencedCache, @@ -14,16 +18,16 @@ from scrapy.utils.python import garbage_collect __doctests__ = ["scrapy.utils.datatypes"] -class CaselessDictTest(unittest.TestCase): +class CaseInsensitiveDictMixin: def test_init_dict(self): seq = {"red": 1, "black": 3} - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) def test_init_pair_sequence(self): seq = (("red", 1), ("black", 3)) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) @@ -42,7 +46,7 @@ class CaselessDictTest(unittest.TestCase): return len(self._d) seq = MyMapping(red=1, black=3) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) @@ -67,12 +71,12 @@ class CaselessDictTest(unittest.TestCase): return len(self._d) seq = MyMutableMapping(red=1, black=3) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) def test_caseless(self): - d = CaselessDict() + d = self.dict_class() d["key_Lower"] = 1 self.assertEqual(d["KEy_loWer"], 1) self.assertEqual(d.get("KEy_loWer"), 1) @@ -82,7 +86,7 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d.get("key_Lower"), 3) def test_delete(self): - d = CaselessDict({"key_lower": 1}) + d = self.dict_class({"key_lower": 1}) del d["key_LOWER"] self.assertRaises(KeyError, d.__getitem__, "key_LOWER") self.assertRaises(KeyError, d.__getitem__, "key_lower") @@ -107,15 +111,15 @@ class CaselessDictTest(unittest.TestCase): def test_fromkeys(self): keys = ("a", "b") - d = CaselessDict.fromkeys(keys) + d = self.dict_class.fromkeys(keys) self.assertEqual(d["A"], None) self.assertEqual(d["B"], None) - d = CaselessDict.fromkeys(keys, 1) + d = self.dict_class.fromkeys(keys, 1) self.assertEqual(d["A"], 1) self.assertEqual(d["B"], 1) - instance = CaselessDict() + instance = self.dict_class() d = instance.fromkeys(keys) self.assertEqual(d["A"], None) self.assertEqual(d["B"], None) @@ -125,31 +129,35 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d["B"], 1) def test_contains(self): - d = CaselessDict() + d = self.dict_class() d["a"] = 1 - assert "a" in d + assert "A" in d def test_pop(self): - d = CaselessDict() + d = self.dict_class() d["a"] = 1 self.assertEqual(d.pop("A"), 1) self.assertRaises(KeyError, d.pop, "A") def test_normkey(self): - class MyDict(CaselessDict): - def normkey(self, key): + class MyDict(self.dict_class): + def _normkey(self, key): return key.title() + normkey = _normkey # deprecated CaselessDict class + d = MyDict() d["key-one"] = 2 self.assertEqual(list(d.keys()), ["Key-One"]) def test_normvalue(self): - class MyDict(CaselessDict): - def normvalue(self, value): + class MyDict(self.dict_class): + def _normvalue(self, value): if value is not None: return value + 1 + normvalue = _normvalue # deprecated CaselessDict class + d = MyDict({"key": 1}) self.assertEqual(d["key"], 2) self.assertEqual(d.get("key"), 2) @@ -174,11 +182,51 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d.get("key"), 2) def test_copy(self): - h1 = CaselessDict({"header1": "value"}) + h1 = self.dict_class({"header1": "value"}) h2 = copy.copy(h1) + assert isinstance(h2, self.dict_class) self.assertEqual(h1, h2) self.assertEqual(h1.get("header1"), h2.get("header1")) - assert isinstance(h2, CaselessDict) + self.assertEqual(h1.get("header1"), h2.get("HEADER1")) + h3 = h1.copy() + assert isinstance(h3, self.dict_class) + self.assertEqual(h1, h3) + self.assertEqual(h1.get("header1"), h3.get("header1")) + self.assertEqual(h1.get("header1"), h3.get("HEADER1")) + + +class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): + dict_class = CaseInsensitiveDict + + def test_repr(self): + d1 = self.dict_class({"foo": "bar"}) + self.assertEqual(repr(d1), "") + d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) + self.assertEqual( + repr(d2), "" + ) + + def test_iter(self): + d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) + iterkeys = iter(d) + self.assertIsInstance(iterkeys, Iterator) + self.assertEqual(list(iterkeys), ["AsDf", "FoO"]) + + +class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): + dict_class = CaselessDict + + def test_deprecation_message(self): + with warnings.catch_warnings(record=True) as caught: + self.dict_class({"foo": "bar"}) + + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) + self.assertEqual( + "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", + str(caught[0].message), + ) class SequenceExcludeTest(unittest.TestCase):