diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 13e636055..b45b12540 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -159,8 +159,9 @@ item_scraped :param spider: the spider which scraped the item :type spider: :class:`~scrapy.Spider` object - :param response: the response from where the item was scraped - :type response: :class:`~scrapy.http.Response` object + :param response: the response from where the item was scraped, or ``None`` + if it was yielded from :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` item_dropped ~~~~~~~~~~~~ @@ -179,8 +180,9 @@ item_dropped :param spider: the spider which scraped the item :type spider: :class:`~scrapy.Spider` object - :param response: the response from where the item was dropped - :type response: :class:`~scrapy.http.Response` object + :param response: the response from where the item was dropped, or ``None`` + if it was yielded from :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` :param exception: the exception (which must be a :exc:`~scrapy.exceptions.DropItem` subclass) which caused the item @@ -201,8 +203,10 @@ item_error :param item: the item that caused the error in the :ref:`topics-item-pipeline` :type item: :ref:`item object ` - :param response: the response being processed when the exception was raised - :type response: :class:`~scrapy.http.Response` object + :param response: the response being processed when the exception was + raised, or ``None`` if it was yielded from + :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8ddf17a14..8f39bcd53 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -176,7 +176,7 @@ object gives you access, for example, to the :ref:`settings `. items). It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects. + return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. .. note:: When implementing this method in your spider middleware, you should always return an iterable (that follows the input one) and diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8a0102a51..e1b1c5ad6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -203,7 +203,8 @@ scrapy.Spider .. method:: start_requests() - This method must return an iterable with the first Requests to crawl for + This method must return an iterable with the first Requests to crawl and/or with :ref:`item objects + ` for this spider. It is called by Scrapy when the spider is opened for scraping. Scrapy calls it only once, so it is safe to implement :meth:`start_requests` as a generator. diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 5318cbd64..63d84339d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -24,6 +24,7 @@ from typing import ( cast, ) +from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -194,7 +195,7 @@ class ExecutionEngine: if self.slot.start_requests is not None and not self._needs_backout(): try: - request = next(self.slot.start_requests) + request_or_item = next(self.slot.start_requests) except StopIteration: self.slot.start_requests = None except Exception: @@ -205,7 +206,16 @@ class ExecutionEngine: extra={"spider": self.spider}, ) else: - self.crawl(request) + if isinstance(request_or_item, Request): + self.crawl(request_or_item) + elif is_item(request_or_item): + self.scraper.start_itemproc(request_or_item, response=None) + else: + logger.error( + f"Got {request_or_item!r} among start requests. Only " + f"requests and items are supported. It will be " + f"ignored." + ) if self.spider_is_idle() and self.slot.close_if_idle: self._spider_idle() diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index a7d65e1e3..7a51dbeb4 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -313,15 +313,11 @@ class Scraper: """Process each Request/Item (given in the output parameter) returned from the given spider """ - assert self.slot is not None # typing if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) elif is_item(output): - self.slot.itemproc_size += 1 - dfd = self.itemproc.process_item(output, spider) - dfd.addBoth(self._itemproc_finished, output, response, spider) - return dfd + return self.start_itemproc(output, response=response) elif output is None: pass else: @@ -333,6 +329,19 @@ class Scraper: ) return None + def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]: + """Send *item* to the item pipelines for processing. + + *response* is the source of the item data. If the item does not come + from response data, e.g. it was hard-coded, set it to ``None``. + """ + assert self.slot is not None # typing + assert self.crawler.spider is not None # typing + self.slot.itemproc_size += 1 + dfd = self.itemproc.process_item(item, self.crawler.spider) + dfd.addBoth(self._itemproc_finished, item, response, self.crawler.spider) + return dfd + def _log_download_errors( self, spider_failure: Failure, @@ -373,7 +382,7 @@ class Scraper: return None def _itemproc_finished( - self, output: Any, item: Any, response: Response, spider: Spider + self, output: Any, item: Any, response: Optional[Response], spider: Spider ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 601209fb0..fea7003e5 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -9,6 +9,7 @@ from twisted.python.failure import Failure # working around https://github.com/sphinx-doc/sphinx/issues/10400 from scrapy import Request, Spider # noqa: TC001 from scrapy.http import Response # noqa: TC001 +from scrapy.utils.python import global_object_name from scrapy.utils.request import referer_str if TYPE_CHECKING: @@ -92,11 +93,13 @@ class LogFormatter: } def scraped( - self, item: Any, response: Union[Response, Failure], spider: Spider + self, item: Any, response: Union[Response, Failure, None], spider: Spider ) -> LogFormatterResult: """Logs a message when an item is scraped by a spider.""" src: Any - if isinstance(response, Failure): + if response is None: + src = f"{global_object_name(spider.__class__)}.start_requests" + elif isinstance(response, Failure): src = response.getErrorMessage() else: src = response @@ -110,7 +113,11 @@ class LogFormatter: } def dropped( - self, item: Any, exception: BaseException, response: Response, spider: Spider + self, + item: Any, + exception: BaseException, + response: Optional[Response], + spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { @@ -123,7 +130,11 @@ class LogFormatter: } def item_error( - self, item: Any, exception: BaseException, response: Response, spider: Spider + self, + item: Any, + exception: BaseException, + response: Optional[Response], + spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing through the item pipeline. diff --git a/tests/spiders.py b/tests/spiders.py index 743811893..5d5792858 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -346,6 +346,19 @@ class BrokenStartRequestsSpider(FollowAllSpider): yield from super().parse(response) +class StartRequestsItemSpider(FollowAllSpider): + def start_requests(self): + yield {"name": "test item"} + + +class StartRequestsGoodAndBadOutput(FollowAllSpider): + def start_requests(self): + yield {"a": "a"} + yield Request("data:,a") + yield "data:,b" + yield object() + + class SingleRequestSpider(MetaSpider): seed = None callback_func = None diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 6cde4ed8c..125709571 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,5 +1,6 @@ import json import logging +import re import unittest from ipaddress import IPv4Address from socket import gethostbyname @@ -49,6 +50,8 @@ from tests.spiders import ( HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, + StartRequestsGoodAndBadOutput, + StartRequestsItemSpider, ) @@ -184,6 +187,39 @@ class CrawlTestCase(TestCase): self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) + @defer.inlineCallbacks + def test_start_requests_items(self): + with LogCapture("scrapy", level=logging.ERROR) as log: + crawler = get_crawler(StartRequestsItemSpider) + yield crawler.crawl(mockserver=self.mockserver) + + self.assertEqual(len(log.records), 0) + + @defer.inlineCallbacks + def test_start_requests_unsupported_output(self): + with LogCapture("scrapy", level=logging.ERROR) as log: + crawler = get_crawler(StartRequestsGoodAndBadOutput) + yield crawler.crawl(mockserver=self.mockserver) + + self.assertEqual(len(log.records), 2) + self.assertEqual( + log.records[0].msg, + ( + "Got 'data:,b' among start requests. Only requests and items " + "are supported. It will be ignored." + ), + ) + self.assertTrue( + re.match( + ( + r"^Got among start " + r"requests\. Only requests and items are supported\. It " + r"will be ignored\.$" + ), + log.records[1].msg, + ) + ) + @defer.inlineCallbacks def test_start_requests_laziness(self): settings = {"CONCURRENT_REQUESTS": 1} diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 38ca8d950..9dbffe353 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,5 +1,5 @@ import collections.abc -from typing import Optional +from typing import Optional, Union from unittest import mock from testfixtures import LogCapture @@ -112,7 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): Should work for process_spider_output and, when it's supported, process_start_requests. """ - ITEM_TYPE: type + ITEM_TYPE: Union[type, tuple] RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod @@ -328,12 +328,13 @@ class ProcessStartRequestsSimpleMiddleware: class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): """process_start_requests tests for simple start_requests""" - ITEM_TYPE = Request + ITEM_TYPE = (Request, dict) MW_SIMPLE = ProcessStartRequestsSimpleMiddleware def _start_requests(self): - for i in range(3): + for i in range(2): yield Request(f"https://example.com/{i}", dont_filter=True) + yield {"name": "test item"} @defer.inlineCallbacks def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):