Merge pull request #5146 from elacuesta/case-insensitive-dict

CaseInsensitiveDict (deprecate CaselessDict)

scrapy/http/headers.py

@@ -2,7 +2,7 @@ from collections.abc import Mapping
 
 from w3lib.http import headers_dict_to_raw
 
-from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict
 from scrapy.utils.python import to_unicode
 
 
@@ -88,7 +88,7 @@ class Headers(CaselessDict):
         """Return headers as a CaselessDict with unicode keys
         and unicode values. Multiple values are joined with ','.
         """
-        return CaselessDict(
+        return CaseInsensitiveDict(
             (
                 to_unicode(key, encoding=self.encoding),
                 to_unicode(b",".join(value), encoding=self.encoding),

scrapy/pipelines/files.py

@@ -28,7 +28,7 @@ from scrapy.http.request import NO_CALLBACK
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
 from scrapy.utils.boto import is_botocore_available
-from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.datatypes import CaseInsensitiveDict
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import md5sum
@@ -155,7 +155,7 @@ class S3FilesStore:
     def _headers_to_botocore_kwargs(self, headers):
         """Convert headers to botocore keyword arguments."""
         # This is required while we need to support both boto and botocore.
-        mapping = CaselessDict(
+        mapping = CaseInsensitiveDict(
             {
                 "Content-Type": "ContentType",
                 "Cache-Control": "CacheControl",

scrapy/utils/datatypes.py

@@ -6,13 +6,29 @@ This module must not depend on any module outside the Standard Library.
 """
 
 import collections
+import warnings
 import weakref
 from collections.abc import Mapping
+from typing import Any, AnyStr
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 class CaselessDict(dict):
     __slots__ = ()
 
+    def __new__(cls, *args, **kwargs):
+        from scrapy.http.headers import Headers
+
+        if issubclass(cls, CaselessDict) and not issubclass(cls, Headers):
+            warnings.warn(
+                "scrapy.utils.datatypes.CaselessDict is deprecated,"
+                " please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+        return super().__new__(cls, *args, **kwargs)
+
     def __init__(self, seq=None):
         super().__init__()
         if seq:
@@ -64,6 +80,48 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
+class CaseInsensitiveDict(collections.UserDict):
+    """A dict-like structure that accepts strings or bytes
+    as keys and allows case-insensitive lookups.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        self._keys: dict = {}
+        super().__init__(*args, **kwargs)
+
+    def __getitem__(self, key: AnyStr) -> Any:
+        normalized_key = self._normkey(key)
+        return super().__getitem__(self._keys[normalized_key.lower()])
+
+    def __setitem__(self, key: AnyStr, value: Any) -> None:
+        normalized_key = self._normkey(key)
+        try:
+            lower_key = self._keys[normalized_key.lower()]
+            del self[lower_key]
+        except KeyError:
+            pass
+        super().__setitem__(normalized_key, self._normvalue(value))
+        self._keys[normalized_key.lower()] = normalized_key
+
+    def __delitem__(self, key: AnyStr) -> None:
+        normalized_key = self._normkey(key)
+        stored_key = self._keys.pop(normalized_key.lower())
+        super().__delitem__(stored_key)
+
+    def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
+        normalized_key = self._normkey(key)
+        return normalized_key.lower() in self._keys
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}: {super().__repr__()}>"
+
+    def _normkey(self, key: AnyStr) -> AnyStr:
+        return key
+
+    def _normvalue(self, value: Any) -> Any:
+        return value
+
+
 class LocalCache(collections.OrderedDict):
     """Dictionary with a finite number of keys.
 

tests/test_utils_datatypes.py

@@ -1,9 +1,13 @@
 import copy
 import unittest
+import warnings
 from collections.abc import Mapping, MutableMapping
+from typing import Iterator
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.utils.datatypes import (
+    CaseInsensitiveDict,
     CaselessDict,
     LocalCache,
     LocalWeakReferencedCache,
@@ -14,16 +18,16 @@ from scrapy.utils.python import garbage_collect
 __doctests__ = ["scrapy.utils.datatypes"]
 
 
-class CaselessDictTest(unittest.TestCase):
+class CaseInsensitiveDictMixin:
     def test_init_dict(self):
         seq = {"red": 1, "black": 3}
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d["red"], 1)
         self.assertEqual(d["black"], 3)
 
     def test_init_pair_sequence(self):
         seq = (("red", 1), ("black", 3))
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d["red"], 1)
         self.assertEqual(d["black"], 3)
 
@@ -42,7 +46,7 @@ class CaselessDictTest(unittest.TestCase):
                 return len(self._d)
 
         seq = MyMapping(red=1, black=3)
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d["red"], 1)
         self.assertEqual(d["black"], 3)
 
@@ -67,12 +71,12 @@ class CaselessDictTest(unittest.TestCase):
                 return len(self._d)
 
         seq = MyMutableMapping(red=1, black=3)
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d["red"], 1)
         self.assertEqual(d["black"], 3)
 
     def test_caseless(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d["key_Lower"] = 1
         self.assertEqual(d["KEy_loWer"], 1)
         self.assertEqual(d.get("KEy_loWer"), 1)
@@ -82,7 +86,7 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d.get("key_Lower"), 3)
 
     def test_delete(self):
-        d = CaselessDict({"key_lower": 1})
+        d = self.dict_class({"key_lower": 1})
         del d["key_LOWER"]
         self.assertRaises(KeyError, d.__getitem__, "key_LOWER")
         self.assertRaises(KeyError, d.__getitem__, "key_lower")
@@ -107,15 +111,15 @@ class CaselessDictTest(unittest.TestCase):
     def test_fromkeys(self):
         keys = ("a", "b")
 
-        d = CaselessDict.fromkeys(keys)
+        d = self.dict_class.fromkeys(keys)
         self.assertEqual(d["A"], None)
         self.assertEqual(d["B"], None)
 
-        d = CaselessDict.fromkeys(keys, 1)
+        d = self.dict_class.fromkeys(keys, 1)
         self.assertEqual(d["A"], 1)
         self.assertEqual(d["B"], 1)
 
-        instance = CaselessDict()
+        instance = self.dict_class()
         d = instance.fromkeys(keys)
         self.assertEqual(d["A"], None)
         self.assertEqual(d["B"], None)
@@ -125,31 +129,35 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d["B"], 1)
 
     def test_contains(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d["a"] = 1
-        assert "a" in d
+        assert "A" in d
 
     def test_pop(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d["a"] = 1
         self.assertEqual(d.pop("A"), 1)
         self.assertRaises(KeyError, d.pop, "A")
 
     def test_normkey(self):
-        class MyDict(CaselessDict):
-            def normkey(self, key):
+        class MyDict(self.dict_class):
+            def _normkey(self, key):
                 return key.title()
 
+            normkey = _normkey  # deprecated CaselessDict class
+
         d = MyDict()
         d["key-one"] = 2
         self.assertEqual(list(d.keys()), ["Key-One"])
 
     def test_normvalue(self):
-        class MyDict(CaselessDict):
-            def normvalue(self, value):
+        class MyDict(self.dict_class):
+            def _normvalue(self, value):
                 if value is not None:
                     return value + 1
 
+            normvalue = _normvalue  # deprecated CaselessDict class
+
         d = MyDict({"key": 1})
         self.assertEqual(d["key"], 2)
         self.assertEqual(d.get("key"), 2)
@@ -174,11 +182,51 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d.get("key"), 2)
 
     def test_copy(self):
-        h1 = CaselessDict({"header1": "value"})
+        h1 = self.dict_class({"header1": "value"})
         h2 = copy.copy(h1)
+        assert isinstance(h2, self.dict_class)
         self.assertEqual(h1, h2)
         self.assertEqual(h1.get("header1"), h2.get("header1"))
-        assert isinstance(h2, CaselessDict)
+        self.assertEqual(h1.get("header1"), h2.get("HEADER1"))
+        h3 = h1.copy()
+        assert isinstance(h3, self.dict_class)
+        self.assertEqual(h1, h3)
+        self.assertEqual(h1.get("header1"), h3.get("header1"))
+        self.assertEqual(h1.get("header1"), h3.get("HEADER1"))
+
+
+class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
+    dict_class = CaseInsensitiveDict
+
+    def test_repr(self):
+        d1 = self.dict_class({"foo": "bar"})
+        self.assertEqual(repr(d1), "<CaseInsensitiveDict: {'foo': 'bar'}>")
+        d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+        self.assertEqual(
+            repr(d2), "<CaseInsensitiveDict: {'AsDf': 'QwErTy', 'FoO': 'bAr'}>"
+        )
+
+    def test_iter(self):
+        d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+        iterkeys = iter(d)
+        self.assertIsInstance(iterkeys, Iterator)
+        self.assertEqual(list(iterkeys), ["AsDf", "FoO"])
+
+
+class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
+    dict_class = CaselessDict
+
+    def test_deprecation_message(self):
+        with warnings.catch_warnings(record=True) as caught:
+            self.dict_class({"foo": "bar"})
+
+            self.assertEqual(len(caught), 1)
+            self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))
+            self.assertEqual(
+                "scrapy.utils.datatypes.CaselessDict is deprecated,"
+                " please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
+                str(caught[0].message),
+            )
 
 
 class SequenceExcludeTest(unittest.TestCase):