Remove or post-pone the use of itemadapter.is_item, as a potentially expensive call (#6719)

docs/faq.rst

@@ -361,16 +361,18 @@ method for this purpose. For example:
 
     from copy import deepcopy
 
-    from itemadapter import is_item, ItemAdapter
+    from itemadapter import ItemAdapter
+    from scrapy import Request
 
 
     class MultiplyItemsMiddleware:
         def process_spider_output(self, response, result, spider):
-            for item in result:
-                if is_item(item):
-                    adapter = ItemAdapter(item)
-                    for _ in range(adapter["multiply_by"]):
-                        yield deepcopy(item)
+            for item_or_request in result:
+                if isinstance(item_or_request, Request):
+                    continue
+                adapter = ItemAdapter(item)
+                for _ in range(adapter["multiply_by"]):
+                    yield deepcopy(item)
 
 Does Scrapy support IPv6 addresses?
 -----------------------------------

docs/topics/items.rst

@@ -384,9 +384,8 @@ Supporting All Item Types
 In code that receives an item, such as methods of :ref:`item pipelines
 <topics-item-pipeline>` or :ref:`spider middlewares
 <topics-spider-middleware>`, it is a good practice to use the
-:class:`~itemadapter.ItemAdapter` class and the
-:func:`~itemadapter.is_item` function to write code that works for
-any supported item type.
+:class:`~itemadapter.ItemAdapter` class to write code that works for any
+supported item type.
 
 Other classes related to items
 ==============================

scrapy/commands/parse.py

@@ -6,7 +6,7 @@ import json
 import logging
 from typing import TYPE_CHECKING, Any, TypeVar, overload
 
-from itemadapter import ItemAdapter, is_item
+from itemadapter import ItemAdapter
 from twisted.internet.defer import Deferred, maybeDeferred
 from w3lib.url import is_url
 
@@ -211,10 +211,10 @@ class Command(BaseRunSpiderCommand):
     ) -> tuple[list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT]:
         items, requests = [], []
         for x in spider_output:
-            if is_item(x):
-                items.append(x)
-            elif isinstance(x, Request):
+            if isinstance(x, Request):
                 requests.append(x)
+            else:
+                items.append(x)
         return items, requests, opts, depth, spider, callback
 
     def run_callback(

scrapy/core/engine.py

@@ -11,7 +11,6 @@ import logging
 from time import time
 from typing import TYPE_CHECKING, Any, TypeVar, cast
 
-from itemadapter import is_item
 from twisted.internet.defer import Deferred, inlineCallbacks, succeed
 from twisted.internet.task import LoopingCall
 from twisted.python.failure import Failure
@@ -194,14 +193,8 @@ class ExecutionEngine:
             else:
                 if isinstance(request_or_item, Request):
                     self.crawl(request_or_item)
-                elif is_item(request_or_item):
-                    self.scraper.start_itemproc(request_or_item, response=None)
                 else:
-                    logger.error(
-                        f"Got {request_or_item!r} among start requests. Only "
-                        f"requests and items are supported. It will be "
-                        f"ignored."
-                    )
+                    self.scraper.start_itemproc(request_or_item, response=None)
 
         if self.spider_is_idle() and self.slot.close_if_idle:
             self._spider_idle()

scrapy/core/scraper.py

@@ -8,7 +8,6 @@ from collections import deque
 from collections.abc import AsyncIterable, Iterator
 from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
 
-from itemadapter import is_item
 from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure
 
@@ -298,17 +297,10 @@ class Scraper:
         if isinstance(output, Request):
             assert self.crawler.engine is not None  # typing
             self.crawler.engine.crawl(request=output)
-        elif is_item(output):
-            return self.start_itemproc(output, response=response)
         elif output is None:
             pass
         else:
-            typename = type(output).__name__
-            logger.error(
-                "Spider must return request, item, or None, got %(typename)r in %(request)s",
-                {"request": request, "typename": typename},
-                extra={"spider": spider},
-            )
+            return self.start_itemproc(output, response=response)
         return None
 
     def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[Any]:

scrapy/exporters.py

@@ -356,12 +356,12 @@ class PythonItemExporter(BaseItemExporter):
     def _serialize_value(self, value: Any) -> Any:
         if isinstance(value, Item):
             return self.export_item(value)
+        if isinstance(value, (str, bytes)):
+            return to_unicode(value, encoding=self.encoding)
         if is_item(value):
             return dict(self._serialize_item(value))
         if is_listlike(value):
             return [self._serialize_value(v) for v in value]
-        if isinstance(value, (str, bytes)):
-            return to_unicode(value, encoding=self.encoding)
         return value
 
     def _serialize_item(self, item: Any) -> Iterable[tuple[str | bytes, Any]]:

scrapy/templates/project/module/middlewares.py.tmpl

@@ -6,7 +6,7 @@
 from scrapy import signals
 
 # useful for handling different item types with a single interface
-from itemadapter import is_item, ItemAdapter
+from itemadapter import ItemAdapter
 
 
 class ${ProjectName}SpiderMiddleware:

scrapy/utils/serialize.py

@@ -28,12 +28,12 @@ class ScrapyJSONEncoder(json.JSONEncoder):
             return str(o)
         if isinstance(o, defer.Deferred):
             return str(o)
-        if is_item(o):
-            return ItemAdapter(o).asdict()
         if isinstance(o, Request):
             return f"<{type(o).__name__} {o.method} {o.url}>"
         if isinstance(o, Response):
             return f"<{type(o).__name__} {o.status} {o.url}>"
+        if is_item(o):
+            return ItemAdapter(o).asdict()
         return super().default(o)
 
 

tests/test_crawl.py

@@ -1,6 +1,5 @@
 import json
 import logging
-import re
 import unittest
 from ipaddress import IPv4Address
 from socket import gethostbyname
@@ -195,23 +194,16 @@ class TestCrawl(TestCase):
 
     @defer.inlineCallbacks
     def test_start_requests_unsupported_output(self):
+        """Anything that is not a request is assumed to be an item, avoiding a
+        potentially expensive call to itemadapter.is_item, and letting instead
+        things fail when ItemAdapter is actually used on the corresponding
+        non-item object."""
+
         with LogCapture("scrapy", level=logging.ERROR) as log:
             crawler = get_crawler(StartRequestsGoodAndBadOutput)
             yield crawler.crawl(mockserver=self.mockserver)
 
-        assert len(log.records) == 2
-        assert log.records[0].msg == (
-            "Got 'data:,b' among start requests. Only requests and items "
-            "are supported. It will be ignored."
-        )
-        assert re.match(
-            (
-                r"^Got <object object at 0x[0-9a-fA-F]+> among start "
-                r"requests\. Only requests and items are supported\. It "
-                r"will be ignored\.$"
-            ),
-            log.records[1].msg,
-        )
+        assert len(log.records) == 0
 
     @defer.inlineCallbacks
     def test_start_requests_laziness(self):