diff --git a/docs/conf.py b/docs/conf.py index fa257dead..34dd5bcb7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -262,6 +262,9 @@ coverage_ignore_pyobjects = [ # details that are not documented. r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.', # methods r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions + + # Never documented before, and deprecated now. + r'^scrapy\.item\.DictItem$', ] diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 60fbc82f8..260f5882c 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -263,3 +263,9 @@ Field objects .. _dict: https://docs.python.org/2/library/stdtypes.html#dict +Other classes related to Item +============================= + +.. autoclass:: BaseItem + +.. autoclass:: ItemMeta diff --git a/scrapy/item.py b/scrapy/item.py index 9d4786788..73b8f54b0 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,13 +4,15 @@ Scrapy Item See documentation in docs/topics/item.rst """ -from abc import ABCMeta -from pprint import pformat -from copy import deepcopy import collections +from abc import ABCMeta +from copy import deepcopy +from pprint import pformat +from warnings import warn import six +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref @@ -21,7 +23,19 @@ else: class BaseItem(object_ref): - """Base class for all scraped items.""" + """Base class for all scraped items. + + In Scrapy, an object is considered an *item* if it is an instance of either + :class:`BaseItem` or :class:`dict`. For example, when the output of a + spider callback is evaluated, only instances of :class:`BaseItem` or + :class:`dict` are passed to :ref:`item pipelines `. + + If you need instances of a custom class to be considered items by Scrapy, + you must inherit from either :class:`BaseItem` or :class:`dict`. + + Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be + :ref:`tracked ` to debug memory leaks. + """ pass @@ -30,6 +44,10 @@ class Field(dict): class ItemMeta(ABCMeta): + """Metaclass_ of :class:`Item` that handles field definitions. + + .. _metaclass: https://realpython.com/python-metaclasses + """ def __new__(mcs, class_name, bases, attrs): classcell = attrs.pop('__classcell__', None) @@ -56,6 +74,13 @@ class DictItem(MutableMapping, BaseItem): fields = {} + def __new__(cls, *args, **kwargs): + if issubclass(cls, DictItem) and not issubclass(cls, Item): + warn('scrapy.item.DictItem is deprecated, please use ' + 'scrapy.item.Item instead', + ScrapyDeprecationWarning, stacklevel=2) + return super(DictItem, cls).__new__(cls, *args, **kwargs) + def __init__(self, *args, **kwargs): self._values = {} if args or kwargs: # avoid creating dict for most common case diff --git a/tests/test_item.py b/tests/test_item.py index 010d3b141..947566686 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,9 +1,11 @@ import sys import unittest +from warnings import catch_warnings import six -from scrapy.item import ABCMeta, Item, ItemMeta, Field +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta from tests import mock @@ -257,6 +259,17 @@ class ItemTest(unittest.TestCase): item['tags'].append('tag2') assert item['tags'] != copied_item['tags'] + def test_dictitem_deprecation_warning(self): + """Make sure the DictItem deprecation warning is not issued for + Item""" + with catch_warnings(record=True) as warnings: + item = Item() + self.assertEqual(len(warnings), 0) + class SubclassedItem(Item): + pass + subclassed_item = SubclassedItem() + self.assertEqual(len(warnings), 0) + class ItemMetaTest(unittest.TestCase): @@ -302,5 +315,20 @@ class ItemMetaClassCellRegression(unittest.TestCase): super(MyItem, self).__init__(*args, **kwargs) +class DictItemTest(unittest.TestCase): + + def test_deprecation_warning(self): + with catch_warnings(record=True) as warnings: + dict_item = DictItem() + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + with catch_warnings(record=True) as warnings: + class SubclassedDictItem(DictItem): + pass + subclassed_dict_item = SubclassedDictItem() + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + + if __name__ == "__main__": unittest.main()