Provide complete API documentation coverage of scrapy.item

docs/conf.py

@@ -262,6 +262,9 @@ coverage_ignore_pyobjects = [
     # details that are not documented.
     r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.',  # methods
     r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]',  # helper functions
+
+    # Never documented before, and deprecated now.
+    r'^scrapy\.item\.DictItem$',
 ]
 
 

docs/topics/items.rst

@@ -263,3 +263,9 @@ Field objects
 .. _dict: https://docs.python.org/2/library/stdtypes.html#dict
 
 
+Other classes related to Item
+=============================
+
+.. autoclass:: BaseItem
+
+.. autoclass:: ItemMeta

scrapy/item.py

@@ -4,13 +4,15 @@ Scrapy Item
 See documentation in docs/topics/item.rst
 """
 
-from abc import ABCMeta
-from pprint import pformat
-from copy import deepcopy
 import collections
+from abc import ABCMeta
+from copy import deepcopy
+from pprint import pformat
+from warnings import warn
 
 import six
 
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
 
@@ -21,7 +23,19 @@ else:
 
 
 class BaseItem(object_ref):
-    """Base class for all scraped items."""
+    """Base class for all scraped items.
+
+    In Scrapy, an object is considered an *item* if it is an instance of either
+    :class:`BaseItem` or :class:`dict`. For example, when the output of a
+    spider callback is evaluated, only instances of :class:`BaseItem` or
+    :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
+
+    If you need instances of a custom class to be considered items by Scrapy,
+    you must inherit from either :class:`BaseItem` or :class:`dict`.
+
+    Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be
+    :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
+    """
     pass
 
 
@@ -30,6 +44,10 @@ class Field(dict):
 
 
 class ItemMeta(ABCMeta):
+    """Metaclass_ of :class:`Item` that handles field definitions.
+
+    .. _metaclass: https://realpython.com/python-metaclasses
+    """
 
     def __new__(mcs, class_name, bases, attrs):
         classcell = attrs.pop('__classcell__', None)
@@ -56,6 +74,13 @@ class DictItem(MutableMapping, BaseItem):
 
     fields = {}
 
+    def __new__(cls, *args, **kwargs):
+        if issubclass(cls, DictItem) and not issubclass(cls, Item):
+            warn('scrapy.item.DictItem is deprecated, please use '
+                 'scrapy.item.Item instead',
+                 ScrapyDeprecationWarning, stacklevel=2)
+        return super(DictItem, cls).__new__(cls, *args, **kwargs)
+
     def __init__(self, *args, **kwargs):
         self._values = {}
         if args or kwargs:  # avoid creating dict for most common case

tests/test_item.py

@@ -1,9 +1,11 @@
 import sys
 import unittest
+from warnings import catch_warnings
 
 import six
 
-from scrapy.item import ABCMeta, Item, ItemMeta, Field
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
 from tests import mock
 
 
@@ -257,6 +259,17 @@ class ItemTest(unittest.TestCase):
         item['tags'].append('tag2')
         assert item['tags'] != copied_item['tags']
 
+    def test_dictitem_deprecation_warning(self):
+        """Make sure the DictItem deprecation warning is not issued for
+        Item"""
+        with catch_warnings(record=True) as warnings:
+            item = Item()
+            self.assertEqual(len(warnings), 0)
+            class SubclassedItem(Item):
+                pass
+            subclassed_item = SubclassedItem()
+            self.assertEqual(len(warnings), 0)
+
 
 class ItemMetaTest(unittest.TestCase):
 
@@ -302,5 +315,20 @@ class ItemMetaClassCellRegression(unittest.TestCase):
                 super(MyItem, self).__init__(*args, **kwargs)
 
 
+class DictItemTest(unittest.TestCase):
+
+    def test_deprecation_warning(self):
+        with catch_warnings(record=True) as warnings:
+            dict_item = DictItem()
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+        with catch_warnings(record=True) as warnings:
+            class SubclassedDictItem(DictItem):
+                pass
+            subclassed_dict_item = SubclassedDictItem()
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
+
 if __name__ == "__main__":
     unittest.main()