Allow raising CloseSpider from seed iterables

docs/news.rst

@@ -84,6 +84,12 @@ New features
     (:issue:`740`, :issue:`1051`, :issue:`1443`, :issue:`3237`, :issue:`4467`,
     :issue:`5282`, :issue:`6730`)
 
+-   You can now raise :exc:`~scrapy.exceptions.CloseSpider` from
+    :meth:`~scrapy.Spider.yield_seeds` and from
+    :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds`.
+
+    (:issue:`3463`, :issue:`4058`, :issue:`6148`, :issue:`6715`, :issue:`6728`)
+
 Bug fixes
 ~~~~~~~~~
 

docs/topics/exceptions.rst

@@ -18,21 +18,7 @@ Here's a list of all exceptions included in Scrapy and their usage.
 CloseSpider
 -----------
 
-.. exception:: CloseSpider(reason='cancelled')
-
-    This exception can be raised from a spider callback to request the spider to be
-    closed/stopped. Supported arguments:
-
-    :param reason: the reason for closing
-    :type reason: str
-
-For example:
-
-.. code-block:: python
-
-    def parse_page(self, response):
-        if "Bandwidth exceeded" in response.body:
-            raise CloseSpider("bandwidth_exceeded")
+.. autoexception:: CloseSpider
 
 DontCloseSpider
 ---------------

docs/topics/spider-middleware.rst

@@ -84,7 +84,8 @@ one or more of these methods:
                     yield seed
 
         You may yield the same type of objects as
-        :meth:`~scrapy.Spider.yield_seeds`.
+        :meth:`~scrapy.Spider.yield_seeds`. It may also raise
+        :exc:`~scrapy.exceptions.CloseSpider`.
 
         As with :meth:`~scrapy.Spider.yield_seeds`, how this method is iterated
         by default is controlled by :setting:`SEEDING_POLICY`. It is also

scrapy/core/engine.py

@@ -193,6 +193,9 @@ class ExecutionEngine:
             seed = yield deferred_from_coro(self._seeds.__anext__())
         except StopAsyncIteration:
             self._seeds = None
+        except CloseSpider:
+            self._seeds = None
+            raise
         except Exception:
             self._seeds = None
             logger.error(
@@ -265,6 +268,9 @@ class ExecutionEngine:
         except _SeedingPolicyChange:
             self._slot.nextcall.schedule()
             return
+        except CloseSpider as exception:
+            self.close_spider(self.spider, reason=exception.reason)
+            return
 
         if self.spider_is_idle() and self._slot.close_if_idle:
             self._spider_idle()

scrapy/exceptions.py

@@ -35,7 +35,30 @@ class DontCloseSpider(Exception):
 
 
 class CloseSpider(Exception):
-    """Raise this from callbacks to request the spider to be closed"""
+    """Raised to request the closing of the running :ref:`spider
+    <topics-spiders>`.
+
+    You may specify a *reason* for closing the spider. It can be an arbitrary
+    string, but it is a best practice to name it like a Python variable name,
+    for example:
+
+    .. code-block:: python
+
+        def parse(self, response):
+            if "Bandwidth exceeded" in response.body:
+                raise CloseSpider("bandwidth_exceeded")
+
+    This exception may be raised from:
+
+    -   The :ref:`callbacks
+        <topics-request-response-ref-request-callback-arguments>` and the
+        :meth:`~scrapy.Spider.yield_seeds` method of spiders.
+
+    -   The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds`
+        method of :ref:`spider middlewares <topics-spider-middleware>`.
+
+    -   :ref:`Handlers <topics-signals>` of the :signal:`spider_idle` signal.
+    """
 
     def __init__(self, reason: str = "cancelled"):
         super().__init__()

scrapy/spiders/__init__.py

@@ -127,6 +127,18 @@ class Spider(object_ref):
                 yield self.crawler.settings["SEEDING_POLICY"]
                 yield Request("https://c.example")
 
+        It is also possible to raise :exc:`~scrapy.exceptions.CloseSpider`, for
+        example to customize the close reason when there are no seeds to yield:
+
+        .. code-block:: python
+
+            async def yield_seeds(self):
+                seeds = await queue_service_client.get_seed_batch()
+                if not seeds:
+                    raise CloseSpider("no_seeds")
+                for seed in seeds:
+                    yield seed
+
         To write spiders that work on Scrapy versions lower than VERSION,
         define also a synchronous ``start_requests()`` method that returns an
         iterable. For example:

tests/test_closespider.py

@@ -1,6 +1,9 @@
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
+from scrapy import Spider, signals
+from scrapy.exceptions import CloseSpider
+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
 from tests.spiders import (
@@ -109,3 +112,43 @@ class TestCloseSpider(TestCase):
         assert reason == "closespider_timeout_no_item"
         total_seconds = crawler.stats.get_value("elapsed_time_seconds")
         assert total_seconds >= timeout
+
+    @deferred_f_from_coro_f
+    async def test_yield_seeds(self):
+        class TestSpider(Spider):
+            name = "test"
+
+            async def yield_seeds(self):
+                raise CloseSpider("foo")
+                yield
+
+        close_reasons = []
+
+        def track_close_reason(spider, reason):
+            close_reasons.append(reason)
+
+        crawler = get_crawler(TestSpider)
+        crawler.signals.connect(track_close_reason, signal=signals.spider_closed)
+        await maybe_deferred_to_future(crawler.crawl())
+        assert len(close_reasons) == 1
+        assert close_reasons[0] == "foo"
+
+    @deferred_f_from_coro_f
+    async def test_callback(self):
+        class TestSpider(Spider):
+            name = "test"
+            start_urls = ["data:,"]
+
+            def parse(self, response):
+                raise CloseSpider("foo")
+
+        close_reasons = []
+
+        def track_close_reason(spider, reason):
+            close_reasons.append(reason)
+
+        crawler = get_crawler(TestSpider)
+        crawler.signals.connect(track_close_reason, signal=signals.spider_closed)
+        await maybe_deferred_to_future(crawler.crawl())
+        assert len(close_reasons) == 1
+        assert close_reasons[0] == "foo"