mirror of https://github.com/scrapy/scrapy.git
Merge pull request #4400 from nyov/obsolete-datatypes
Remove deprecated SiteNode and MultiValueDict classes
This commit is contained in:
Mikhail Korobov
GitHub
commit
29f957d8cb
|
|
@ -6,183 +6,9 @@ This module must not depend on any module outside the Standard Library.
|
|||
"""
|
||||
|
||||
import collections
|
||||
import copy
|
||||
import warnings
|
||||
import weakref
|
||||
from collections.abc import Mapping
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
class MultiValueDictKeyError(KeyError):
|
||||
def __init__(self, *args, **kwargs):
|
||||
warnings.warn(
|
||||
"scrapy.utils.datatypes.MultiValueDictKeyError is deprecated "
|
||||
"and will be removed in future releases.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
super(MultiValueDictKeyError, self).__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class MultiValueDict(dict):
|
||||
"""
|
||||
A subclass of dictionary customized to handle multiple values for the same key.
|
||||
|
||||
>>> d = MultiValueDict({'name': ['Adrian', 'Simon'], 'position': ['Developer']})
|
||||
>>> d['name']
|
||||
'Simon'
|
||||
>>> d.getlist('name')
|
||||
['Adrian', 'Simon']
|
||||
>>> d.get('lastname', 'nonexistent')
|
||||
'nonexistent'
|
||||
>>> d.setlist('lastname', ['Holovaty', 'Willison'])
|
||||
|
||||
This class exists to solve the irritating problem raised by cgi.parse_qs,
|
||||
which returns a list for every key, even though most Web forms submit
|
||||
single name-value pairs.
|
||||
"""
|
||||
def __init__(self, key_to_list_mapping=()):
|
||||
warnings.warn("scrapy.utils.datatypes.MultiValueDict is deprecated "
|
||||
"and will be removed in future releases.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2)
|
||||
dict.__init__(self, key_to_list_mapping)
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s: %s>" % (self.__class__.__name__, dict.__repr__(self))
|
||||
|
||||
def __getitem__(self, key):
|
||||
"""
|
||||
Returns the last data value for this key, or [] if it's an empty list;
|
||||
raises KeyError if not found.
|
||||
"""
|
||||
try:
|
||||
list_ = dict.__getitem__(self, key)
|
||||
except KeyError:
|
||||
raise MultiValueDictKeyError("Key %r not found in %r" % (key, self))
|
||||
try:
|
||||
return list_[-1]
|
||||
except IndexError:
|
||||
return []
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
dict.__setitem__(self, key, [value])
|
||||
|
||||
def __copy__(self):
|
||||
return self.__class__(dict.items(self))
|
||||
|
||||
def __deepcopy__(self, memo=None):
|
||||
if memo is None:
|
||||
memo = {}
|
||||
result = self.__class__()
|
||||
memo[id(self)] = result
|
||||
for key, value in dict.items(self):
|
||||
dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo))
|
||||
return result
|
||||
|
||||
def get(self, key, default=None):
|
||||
"Returns the default value if the requested data doesn't exist"
|
||||
try:
|
||||
val = self[key]
|
||||
except KeyError:
|
||||
return default
|
||||
if val == []:
|
||||
return default
|
||||
return val
|
||||
|
||||
def getlist(self, key):
|
||||
"Returns an empty list if the requested data doesn't exist"
|
||||
try:
|
||||
return dict.__getitem__(self, key)
|
||||
except KeyError:
|
||||
return []
|
||||
|
||||
def setlist(self, key, list_):
|
||||
dict.__setitem__(self, key, list_)
|
||||
|
||||
def setdefault(self, key, default=None):
|
||||
if key not in self:
|
||||
self[key] = default
|
||||
return self[key]
|
||||
|
||||
def setlistdefault(self, key, default_list=()):
|
||||
if key not in self:
|
||||
self.setlist(key, default_list)
|
||||
return self.getlist(key)
|
||||
|
||||
def appendlist(self, key, value):
|
||||
"Appends an item to the internal list associated with key"
|
||||
self.setlistdefault(key, [])
|
||||
dict.__setitem__(self, key, self.getlist(key) + [value])
|
||||
|
||||
def items(self):
|
||||
"""
|
||||
Returns a list of (key, value) pairs, where value is the last item in
|
||||
the list associated with the key.
|
||||
"""
|
||||
return [(key, self[key]) for key in self.keys()]
|
||||
|
||||
def lists(self):
|
||||
"Returns a list of (key, list) pairs."
|
||||
return dict.items(self)
|
||||
|
||||
def values(self):
|
||||
"Returns a list of the last value on every key list."
|
||||
return [self[key] for key in self.keys()]
|
||||
|
||||
def copy(self):
|
||||
"Returns a copy of this object."
|
||||
return self.__deepcopy__()
|
||||
|
||||
def update(self, *args, **kwargs):
|
||||
"update() extends rather than replaces existing key lists. Also accepts keyword args."
|
||||
if len(args) > 1:
|
||||
raise TypeError("update expected at most 1 arguments, got %d" % len(args))
|
||||
if args:
|
||||
other_dict = args[0]
|
||||
if isinstance(other_dict, MultiValueDict):
|
||||
for key, value_list in other_dict.lists():
|
||||
self.setlistdefault(key, []).extend(value_list)
|
||||
else:
|
||||
try:
|
||||
for key, value in other_dict.items():
|
||||
self.setlistdefault(key, []).append(value)
|
||||
except TypeError:
|
||||
raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
|
||||
for key, value in kwargs.items():
|
||||
self.setlistdefault(key, []).append(value)
|
||||
|
||||
|
||||
class SiteNode(object):
|
||||
"""Class to represent a site node (page, image or any other file)"""
|
||||
|
||||
def __init__(self, url):
|
||||
warnings.warn(
|
||||
"scrapy.utils.datatypes.SiteNode is deprecated "
|
||||
"and will be removed in future releases.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
|
||||
self.url = url
|
||||
self.itemnames = []
|
||||
self.children = []
|
||||
self.parent = None
|
||||
|
||||
def add_child(self, node):
|
||||
self.children.append(node)
|
||||
node.parent = self
|
||||
|
||||
def to_string(self, level=0):
|
||||
s = "%s%s\n" % (' ' * level, self.url)
|
||||
if self.itemnames:
|
||||
for n in self.itemnames:
|
||||
s += "%sScraped: %s\n" % (' ' * (level + 1), n)
|
||||
for node in self.children:
|
||||
s += node.to_string(level + 1)
|
||||
return s
|
||||
|
||||
|
||||
class CaselessDict(dict):
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue