diff --git a/docs/faq.rst b/docs/faq.rst index cef3e69f3..da255f29e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -361,16 +361,18 @@ method for this purpose. For example: from copy import deepcopy - from itemadapter import is_item, ItemAdapter + from itemadapter import ItemAdapter + from scrapy import Request class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): - for item in result: - if is_item(item): - adapter = ItemAdapter(item) - for _ in range(adapter["multiply_by"]): - yield deepcopy(item) + for item_or_request in result: + if isinstance(item_or_request, Request): + continue + adapter = ItemAdapter(item) + for _ in range(adapter["multiply_by"]): + yield deepcopy(item) Does Scrapy support IPv6 addresses? ----------------------------------- diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 7cc476863..0365c95b3 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -384,9 +384,8 @@ Supporting All Item Types In code that receives an item, such as methods of :ref:`item pipelines ` or :ref:`spider middlewares `, it is a good practice to use the -:class:`~itemadapter.ItemAdapter` class and the -:func:`~itemadapter.is_item` function to write code that works for -any supported item type. +:class:`~itemadapter.ItemAdapter` class to write code that works for any +supported item type. Other classes related to items ============================== diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 61aea3ee4..c6ed20b3b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -6,7 +6,7 @@ import json import logging from typing import TYPE_CHECKING, Any, TypeVar, overload -from itemadapter import ItemAdapter, is_item +from itemadapter import ItemAdapter from twisted.internet.defer import Deferred, maybeDeferred from w3lib.url import is_url @@ -211,10 +211,10 @@ class Command(BaseRunSpiderCommand): ) -> tuple[list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: - if is_item(x): - items.append(x) - elif isinstance(x, Request): + if isinstance(x, Request): requests.append(x) + else: + items.append(x) return items, requests, opts, depth, spider, callback def run_callback( diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 61f444e31..b7a73700b 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -11,7 +11,6 @@ import logging from time import time from typing import TYPE_CHECKING, Any, TypeVar, cast -from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -194,14 +193,8 @@ class ExecutionEngine: else: if isinstance(request_or_item, Request): self.crawl(request_or_item) - elif is_item(request_or_item): - self.scraper.start_itemproc(request_or_item, response=None) else: - logger.error( - f"Got {request_or_item!r} among start requests. Only " - f"requests and items are supported. It will be " - f"ignored." - ) + self.scraper.start_itemproc(request_or_item, response=None) if self.spider_is_idle() and self.slot.close_if_idle: self._spider_idle() diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 03301717d..b664b61f6 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -8,7 +8,6 @@ from collections import deque from collections.abc import AsyncIterable, Iterator from typing import TYPE_CHECKING, Any, TypeVar, Union, cast -from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -298,17 +297,10 @@ class Scraper: if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) - elif is_item(output): - return self.start_itemproc(output, response=response) elif output is None: pass else: - typename = type(output).__name__ - logger.error( - "Spider must return request, item, or None, got %(typename)r in %(request)s", - {"request": request, "typename": typename}, - extra={"spider": spider}, - ) + return self.start_itemproc(output, response=response) return None def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[Any]: diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 46c6aa3fa..0a641752e 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -356,12 +356,12 @@ class PythonItemExporter(BaseItemExporter): def _serialize_value(self, value: Any) -> Any: if isinstance(value, Item): return self.export_item(value) + if isinstance(value, (str, bytes)): + return to_unicode(value, encoding=self.encoding) if is_item(value): return dict(self._serialize_item(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - if isinstance(value, (str, bytes)): - return to_unicode(value, encoding=self.encoding) return value def _serialize_item(self, item: Any) -> Iterable[tuple[str | bytes, Any]]: diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 8c9a86dce..dcb2d63de 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -6,7 +6,7 @@ from scrapy import signals # useful for handling different item types with a single interface -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter class ${ProjectName}SpiderMiddleware: diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 308e351c6..bcfae0c00 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -28,12 +28,12 @@ class ScrapyJSONEncoder(json.JSONEncoder): return str(o) if isinstance(o, defer.Deferred): return str(o) - if is_item(o): - return ItemAdapter(o).asdict() if isinstance(o, Request): return f"<{type(o).__name__} {o.method} {o.url}>" if isinstance(o, Response): return f"<{type(o).__name__} {o.status} {o.url}>" + if is_item(o): + return ItemAdapter(o).asdict() return super().default(o) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5766f9313..6f4045fc8 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,5 @@ import json import logging -import re import unittest from ipaddress import IPv4Address from socket import gethostbyname @@ -195,23 +194,16 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_requests_unsupported_output(self): + """Anything that is not a request is assumed to be an item, avoiding a + potentially expensive call to itemadapter.is_item, and letting instead + things fail when ItemAdapter is actually used on the corresponding + non-item object.""" + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(StartRequestsGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) - assert len(log.records) == 2 - assert log.records[0].msg == ( - "Got 'data:,b' among start requests. Only requests and items " - "are supported. It will be ignored." - ) - assert re.match( - ( - r"^Got among start " - r"requests\. Only requests and items are supported\. It " - r"will be ignored\.$" - ), - log.records[1].msg, - ) + assert len(log.records) == 0 @defer.inlineCallbacks def test_start_requests_laziness(self):