RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5398 from elacuesta/remove-deprecated-item-classes 

Remove deprecated DictItem/BaseItem classes
This commit is contained in:
Andrey Rahmatullin 2022-02-07 12:34:35 +05:00 committed by GitHub
parent 3533717488 fca49cca92
commit f7bf7414f0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 29 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/conf.py
View File

@ -272,7 +272,6 @@ coverage_ignore_pyobjects = [
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
# Never documented before, and deprecated now.
r'^scrapy\.item\.DictItem$',
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
# Implementation detail of LxmlLinkExtractor

4
scrapy/exporters.py
View File

@ -13,7 +13,7 @@ from xml.sax.saxutils import XMLGenerator
from itemadapter import is_item, ItemAdapter
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import _BaseItem
from scrapy.item import Item
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
from scrapy.utils.serialize import ScrapyJSONEncoder
@ -315,7 +315,7 @@ class PythonItemExporter(BaseItemExporter):
return serializer(value)
def _serialize_value(self, value):
if isinstance(value, _BaseItem):
if isinstance(value, Item):
return self.export_item(value)
elif is_item(value):
return dict(self._serialize_item(value))

87
scrapy/item.py
View File

@ -9,45 +9,15 @@ from collections.abc import MutableMapping
from copy import deepcopy
from pprint import pformat
from typing import Dict
from warnings import warn
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.trackref import object_ref
class _BaseItem(object_ref):
"""
Temporary class used internally to avoid the deprecation
warning raised by isinstance checks using BaseItem.
"""
pass
class _BaseItemMeta(ABCMeta):
def __instancecheck__(cls, instance):
if cls is BaseItem:
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super().__instancecheck__(instance)
class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
"""
Deprecated, please use :class:`scrapy.item.Item` instead
"""
def __new__(cls, *args, **kwargs):
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super().__new__(cls, *args, **kwargs)
class Field(dict):
"""Container of field metadata"""
class ItemMeta(_BaseItemMeta):
class ItemMeta(ABCMeta):
"""Metaclass_ of :class:`Item` that handles field definitions.
.. _metaclass: https://realpython.com/python-metaclasses
@ -74,15 +44,30 @@ class ItemMeta(_BaseItemMeta):
return super().__new__(mcs, class_name, bases, new_attrs)
class DictItem(MutableMapping, BaseItem):
class Item(MutableMapping, object_ref, metaclass=ItemMeta):
"""
Base class for scraped items.
fields: Dict[str, Field] = {}
In Scrapy, an object is considered an ``item`` if it is an instance of either
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
spider callback is evaluated, only instances of :class:`Item` or
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
def __new__(cls, *args, **kwargs):
if issubclass(cls, DictItem) and not issubclass(cls, Item):
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super().__new__(cls, *args, **kwargs)
If you need instances of a custom class to be considered items by Scrapy,
you must inherit from either :class:`Item` or :class:`dict`.
Items must declare :class:`Field` attributes, which are processed and stored
in the ``fields`` attribute. This restricts the set of allowed field names
and prevents typos, raising ``KeyError`` when referring to undefined fields.
Additionally, fields can be used to define metadata and control the way
data is processed internally. Please refer to the :ref:`documentation
about fields <topics-items-fields>` for additional information.
Unlike instances of :class:`dict`, instances of :class:`Item` may be
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
"""
fields: Dict[str, Field]
def __init__(self, *args, **kwargs):
self._values = {}
@ -118,7 +103,7 @@ class DictItem(MutableMapping, BaseItem):
def __iter__(self):
return iter(self._values)
__hash__ = BaseItem.__hash__
__hash__ = object_ref.__hash__
def keys(self):
return self._values.keys()
@ -133,27 +118,3 @@ class DictItem(MutableMapping, BaseItem):
"""Return a :func:`~copy.deepcopy` of this item.
"""
return deepcopy(self)
class Item(DictItem, metaclass=ItemMeta):
"""
Base class for scraped items.
In Scrapy, an object is considered an ``item`` if it is an instance of either
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
spider callback is evaluated, only instances of :class:`Item` or
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
If you need instances of a custom class to be considered items by Scrapy,
you must inherit from either :class:`Item` or :class:`dict`.
Items must declare :class:`Field` attributes, which are processed and stored
in the ``fields`` attribute. This restricts the set of allowed field names
and prevents typos, raising ``KeyError`` when referring to undefined fields.
Additionally, fields can be used to define metadata and control the way
data is processed internally. Please refer to the :ref:`documentation
about fields <topics-items-fields>` for additional information.
Unlike instances of :class:`dict`, instances of :class:`Item` may be
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
"""

4
scrapy/utils/misc.py
View File

@ -14,11 +14,11 @@ from w3lib.html import replace_entities
from scrapy.utils.datatypes import LocalWeakReferencedCache
from scrapy.utils.python import flatten, to_unicode
from scrapy.item import _BaseItem
from scrapy.item import Item
from scrapy.utils.deprecate import ScrapyDeprecationWarning
_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
def arg_to_iter(arg):

105
tests/test_item.py
View File

@ -1,9 +1,7 @@
import unittest
from unittest import mock
from warnings import catch_warnings, filterwarnings
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta
from scrapy.item import ABCMeta, Field, Item, ItemMeta
class ItemTest(unittest.TestCase):
@ -254,18 +252,6 @@ class ItemTest(unittest.TestCase):
item['tags'].append('tag2')
assert item['tags'] != copied_item['tags']
def test_dictitem_deprecation_warning(self):
"""Make sure the DictItem deprecation warning is not issued for
Item"""
with catch_warnings(record=True) as warnings:
Item()
self.assertEqual(len(warnings), 0)
class SubclassedItem(Item):
pass
SubclassedItem()
self.assertEqual(len(warnings), 0)
class ItemMetaTest(unittest.TestCase):
@ -303,94 +289,5 @@ class ItemMetaClassCellRegression(unittest.TestCase):
super().__init__(*args, **kwargs)
class DictItemTest(unittest.TestCase):
def test_deprecation_warning(self):
with catch_warnings(record=True) as warnings:
DictItem()
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
with catch_warnings(record=True) as warnings:
class SubclassedDictItem(DictItem):
pass
SubclassedDictItem()
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
class BaseItemTest(unittest.TestCase):
def test_isinstance_check(self):
class SubclassedBaseItem(BaseItem):
pass
class SubclassedItem(Item):
pass
with catch_warnings():
filterwarnings("ignore", category=ScrapyDeprecationWarning)
self.assertTrue(isinstance(BaseItem(), BaseItem))
self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
self.assertTrue(isinstance(Item(), BaseItem))
self.assertTrue(isinstance(SubclassedItem(), BaseItem))
# make sure internal checks using private _BaseItem class succeed
self.assertTrue(isinstance(BaseItem(), _BaseItem))
self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
self.assertTrue(isinstance(Item(), _BaseItem))
self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
def test_deprecation_warning(self):
"""
Make sure deprecation warnings are logged whenever BaseItem is used,
either instantiated or in an isinstance check
"""
with catch_warnings(record=True) as warnings:
BaseItem()
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
with catch_warnings(record=True) as warnings:
class SubclassedBaseItem(BaseItem):
pass
SubclassedBaseItem()
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
with catch_warnings(record=True) as warnings:
self.assertFalse(isinstance("foo", BaseItem))
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
with catch_warnings(record=True) as warnings:
self.assertTrue(isinstance(BaseItem(), BaseItem))
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
class ItemNoDeprecationWarningTest(unittest.TestCase):
def test_no_deprecation_warning(self):
"""
Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used.
"""
class SubclassedItem(Item):
pass
with catch_warnings(record=True) as warnings:
Item()
SubclassedItem()
_BaseItem()
self.assertFalse(isinstance("foo", _BaseItem))
self.assertFalse(isinstance("foo", Item))
self.assertFalse(isinstance("foo", SubclassedItem))
self.assertTrue(isinstance(_BaseItem(), _BaseItem))
self.assertTrue(isinstance(Item(), Item))
self.assertTrue(isinstance(SubclassedItem(), SubclassedItem))
self.assertEqual(len(warnings), 0)
if __name__ == "__main__":
unittest.main()