RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CaseInsensitiveDict (deprecate CaselessDict)

This commit is contained in:
Eugenio Lacuesta 2021-05-12 13:08:08 -03:00
parent ee682af3b0
commit 52d0df5f98
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
4 changed files with 107 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scrapy/http/headers.py
View File

@ -1,5 +1,6 @@
from w3lib.http import headers_dict_to_raw
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict
from scrapy.utils.python import to_unicode
@ -76,13 +77,13 @@ class Headers(CaselessDict):
return headers_dict_to_raw(self)
def to_unicode_dict(self):
""" Return headers as a CaselessDict with unicode keys
""" Return headers as a CaseInsensitiveDict with unicode keys
and unicode values. Multiple values are joined with ','.
"""
return CaselessDict(
(to_unicode(key, encoding=self.encoding),
to_unicode(b','.join(value), encoding=self.encoding))
for key, value in self.items())
return CaseInsensitiveDict(
(to_unicode(key, encoding=self.encoding), to_unicode(b','.join(value), encoding=self.encoding))
for key, value in self.items()
)
def __copy__(self):
return self.__class__(self)

4
scrapy/pipelines/files.py
View File

@ -23,7 +23,7 @@ from scrapy.http import Request
from scrapy.pipelines.media import MediaPipeline
from scrapy.settings import Settings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.datatypes import CaseInsensitiveDict
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import md5sum
@ -143,7 +143,7 @@ class S3FilesStore:
""" Convert headers to botocore keyword agruments.
"""
# This is required while we need to support both boto and botocore.
mapping = CaselessDict({
mapping = CaseInsensitiveDict({
'Content-Type': 'ContentType',
'Cache-Control': 'CacheControl',
'Content-Disposition': 'ContentDisposition',

46
scrapy/utils/datatypes.py
View File

@ -6,14 +6,30 @@ This module must not depend on any module outside the Standard Library.
"""
import collections
import warnings
import weakref
from collections.abc import Mapping
from typing import Any, AnyStr
from scrapy.exceptions import ScrapyDeprecationWarning
class CaselessDict(dict):
__slots__ = ()
def __new__(cls, *args, **kwargs):
from scrapy.http.headers import Headers
if issubclass(cls, CaselessDict) and not issubclass(cls, Headers):
warnings.warn(
"scrapy.utils.datatypes.CaselessDict is deprecated,"
" please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return super().__new__(cls, *args, **kwargs)
def __init__(self, seq=None):
super().__init__()
if seq:
@ -63,6 +79,36 @@ class CaselessDict(dict):
return dict.pop(self, self.normkey(key), *args)
class CaseInsensitiveDict(collections.UserDict):
"""A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups.
It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods.
"""
def __getitem__(self, key: AnyStr) -> Any:
return super().__getitem__(self.normkey(key))
def __setitem__(self, key: AnyStr, value: Any) -> None:
super().__setitem__(self.normkey(key), self.normvalue(value))
def __delitem__(self, key: AnyStr) -> None:
super().__delitem__(self.normkey(key))
def __contains__(self, key: AnyStr) -> bool: # type: ignore[override]
return super().__contains__(self.normkey(key))
def normkey(self, key: AnyStr) -> AnyStr:
"""Method to normalize dictionary key access"""
return key.lower()
def normvalue(self, value: Any) -> Any:
"""Method to normalize values prior to be set"""
return value
def __repr__(self) -> str:
return f"<{self.__class__.__name__}: {super().__repr__()}>"
class LocalCache(collections.OrderedDict):
"""Dictionary with a finite number of keys.

71
tests/test_utils_datatypes.py
View File

@ -1,26 +1,34 @@
import copy
import unittest
import warnings
from collections.abc import Mapping, MutableMapping
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.utils.datatypes import CaselessDict, LocalCache, LocalWeakReferencedCache, SequenceExclude
from scrapy.utils.datatypes import (
CaseInsensitiveDict,
CaselessDict,
LocalCache,
LocalWeakReferencedCache,
SequenceExclude,
)
from scrapy.utils.python import garbage_collect
__doctests__ = ['scrapy.utils.datatypes']
class CaselessDictTest(unittest.TestCase):
class CaseInsensitiveDictMixin:
def test_init_dict(self):
seq = {'red': 1, 'black': 3}
d = CaselessDict(seq)
d = self.dict_class(seq)
self.assertEqual(d['red'], 1)
self.assertEqual(d['black'], 3)
def test_init_pair_sequence(self):
seq = (('red', 1), ('black', 3))
d = CaselessDict(seq)
d = self.dict_class(seq)
self.assertEqual(d['red'], 1)
self.assertEqual(d['black'], 3)
@ -39,7 +47,7 @@ class CaselessDictTest(unittest.TestCase):
return len(self._d)
seq = MyMapping(red=1, black=3)
d = CaselessDict(seq)
d = self.dict_class(seq)
self.assertEqual(d['red'], 1)
self.assertEqual(d['black'], 3)
@ -64,12 +72,12 @@ class CaselessDictTest(unittest.TestCase):
return len(self._d)
seq = MyMutableMapping(red=1, black=3)
d = CaselessDict(seq)
d = self.dict_class(seq)
self.assertEqual(d['red'], 1)
self.assertEqual(d['black'], 3)
def test_caseless(self):
d = CaselessDict()
d = self.dict_class()
d['key_Lower'] = 1
self.assertEqual(d['KEy_loWer'], 1)
self.assertEqual(d.get('KEy_loWer'), 1)
@ -79,19 +87,19 @@ class CaselessDictTest(unittest.TestCase):
self.assertEqual(d.get('key_Lower'), 3)
def test_delete(self):
d = CaselessDict({'key_lower': 1})
d = self.dict_class({'key_lower': 1})
del d['key_LOWER']
self.assertRaises(KeyError, d.__getitem__, 'key_LOWER')
self.assertRaises(KeyError, d.__getitem__, 'key_lower')
def test_getdefault(self):
d = CaselessDict()
d = self.dict_class()
self.assertEqual(d.get('c', 5), 5)
d['c'] = 10
self.assertEqual(d.get('c', 5), 10)
def test_setdefault(self):
d = CaselessDict({'a': 1, 'b': 2})
d = self.dict_class({'a': 1, 'b': 2})
r = d.setdefault('A', 5)
self.assertEqual(r, 1)
@ -104,15 +112,15 @@ class CaselessDictTest(unittest.TestCase):
def test_fromkeys(self):
keys = ('a', 'b')
d = CaselessDict.fromkeys(keys)
d = self.dict_class.fromkeys(keys)
self.assertEqual(d['A'], None)
self.assertEqual(d['B'], None)
d = CaselessDict.fromkeys(keys, 1)
d = self.dict_class.fromkeys(keys, 1)
self.assertEqual(d['A'], 1)
self.assertEqual(d['B'], 1)
instance = CaselessDict()
instance = self.dict_class()
d = instance.fromkeys(keys)
self.assertEqual(d['A'], None)
self.assertEqual(d['B'], None)
@ -122,18 +130,19 @@ class CaselessDictTest(unittest.TestCase):
self.assertEqual(d['B'], 1)
def test_contains(self):
d = CaselessDict()
d = self.dict_class()
d['a'] = 1
assert 'a' in d
assert 'A' in d
def test_pop(self):
d = CaselessDict()
d = self.dict_class()
d['a'] = 1
self.assertEqual(d.pop('A'), 1)
self.assertRaises(KeyError, d.pop, 'A')
def test_normkey(self):
class MyDict(CaselessDict):
class MyDict(self.dict_class):
def normkey(self, key):
return key.title()
@ -142,7 +151,7 @@ class CaselessDictTest(unittest.TestCase):
self.assertEqual(list(d.keys()), ['Key-One'])
def test_normvalue(self):
class MyDict(CaselessDict):
class MyDict(self.dict_class):
def normvalue(self, value):
if value is not None:
return value + 1
@ -171,11 +180,35 @@ class CaselessDictTest(unittest.TestCase):
self.assertEqual(d.get('key'), 2)
def test_copy(self):
h1 = CaselessDict({'header1': 'value'})
h1 = self.dict_class({'header1': 'value'})
h2 = copy.copy(h1)
self.assertEqual(h1, h2)
self.assertEqual(h1.get('header1'), h2.get('header1'))
assert isinstance(h2, CaselessDict)
assert isinstance(h2, self.dict_class)
class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
dict_class = CaseInsensitiveDict
def test_repr(self):
d = self.dict_class({"foo": "bar"})
self.assertEqual(repr(d), "<CaseInsensitiveDict: {'foo': 'bar'}>")
class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
dict_class = CaselessDict
def test_deprecation_message(self):
with warnings.catch_warnings(record=True) as caught:
self.dict_class({"foo": "bar"})
self.assertEqual(len(caught), 1)
self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))
self.assertEqual(
"scrapy.utils.datatypes.CaselessDict is deprecated,"
" please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
str(caught[0].message),
)
class SequenceExcludeTest(unittest.TestCase):