diff --git a/docs/news.rst b/docs/news.rst index 7e8f59b1a..79e7e1365 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -84,6 +84,12 @@ New features (:issue:`740`, :issue:`1051`, :issue:`1443`, :issue:`3237`, :issue:`4467`, :issue:`5282`, :issue:`6730`) +- You can now raise :exc:`~scrapy.exceptions.CloseSpider` from + :meth:`~scrapy.Spider.yield_seeds` and from + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds`. + + (:issue:`3463`, :issue:`4058`, :issue:`6148`, :issue:`6715`, :issue:`6728`) + Bug fixes ~~~~~~~~~ diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 0b572ff95..4eda0c4cc 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -18,21 +18,7 @@ Here's a list of all exceptions included in Scrapy and their usage. CloseSpider ----------- -.. exception:: CloseSpider(reason='cancelled') - - This exception can be raised from a spider callback to request the spider to be - closed/stopped. Supported arguments: - - :param reason: the reason for closing - :type reason: str - -For example: - -.. code-block:: python - - def parse_page(self, response): - if "Bandwidth exceeded" in response.body: - raise CloseSpider("bandwidth_exceeded") +.. autoexception:: CloseSpider DontCloseSpider --------------- diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 9ea46fc53..d57fada99 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -84,7 +84,8 @@ one or more of these methods: yield seed You may yield the same type of objects as - :meth:`~scrapy.Spider.yield_seeds`. + :meth:`~scrapy.Spider.yield_seeds`. It may also raise + :exc:`~scrapy.exceptions.CloseSpider`. As with :meth:`~scrapy.Spider.yield_seeds`, how this method is iterated by default is controlled by :setting:`SEEDING_POLICY`. It is also diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 8383b9395..ddfbb8fba 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -193,6 +193,9 @@ class ExecutionEngine: seed = yield deferred_from_coro(self._seeds.__anext__()) except StopAsyncIteration: self._seeds = None + except CloseSpider: + self._seeds = None + raise except Exception: self._seeds = None logger.error( @@ -265,6 +268,9 @@ class ExecutionEngine: except _SeedingPolicyChange: self._slot.nextcall.schedule() return + except CloseSpider as exception: + self.close_spider(self.spider, reason=exception.reason) + return if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index f37f881a7..e3a652728 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -35,7 +35,30 @@ class DontCloseSpider(Exception): class CloseSpider(Exception): - """Raise this from callbacks to request the spider to be closed""" + """Raised to request the closing of the running :ref:`spider + `. + + You may specify a *reason* for closing the spider. It can be an arbitrary + string, but it is a best practice to name it like a Python variable name, + for example: + + .. code-block:: python + + def parse(self, response): + if "Bandwidth exceeded" in response.body: + raise CloseSpider("bandwidth_exceeded") + + This exception may be raised from: + + - The :ref:`callbacks + ` and the + :meth:`~scrapy.Spider.yield_seeds` method of spiders. + + - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds` + method of :ref:`spider middlewares `. + + - :ref:`Handlers ` of the :signal:`spider_idle` signal. + """ def __init__(self, reason: str = "cancelled"): super().__init__() diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index d2941c7c3..a2cde64f5 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -127,6 +127,18 @@ class Spider(object_ref): yield self.crawler.settings["SEEDING_POLICY"] yield Request("https://c.example") + It is also possible to raise :exc:`~scrapy.exceptions.CloseSpider`, for + example to customize the close reason when there are no seeds to yield: + + .. code-block:: python + + async def yield_seeds(self): + seeds = await queue_service_client.get_seed_batch() + if not seeds: + raise CloseSpider("no_seeds") + for seed in seeds: + yield seed + To write spiders that work on Scrapy versions lower than VERSION, define also a synchronous ``start_requests()`` method that returns an iterable. For example: diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 476662789..d30c99ac2 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,6 +1,9 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase +from scrapy import Spider, signals +from scrapy.exceptions import CloseSpider +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import ( @@ -109,3 +112,43 @@ class TestCloseSpider(TestCase): assert reason == "closespider_timeout_no_item" total_seconds = crawler.stats.get_value("elapsed_time_seconds") assert total_seconds >= timeout + + @deferred_f_from_coro_f + async def test_yield_seeds(self): + class TestSpider(Spider): + name = "test" + + async def yield_seeds(self): + raise CloseSpider("foo") + yield + + close_reasons = [] + + def track_close_reason(spider, reason): + close_reasons.append(reason) + + crawler = get_crawler(TestSpider) + crawler.signals.connect(track_close_reason, signal=signals.spider_closed) + await maybe_deferred_to_future(crawler.crawl()) + assert len(close_reasons) == 1 + assert close_reasons[0] == "foo" + + @deferred_f_from_coro_f + async def test_callback(self): + class TestSpider(Spider): + name = "test" + start_urls = ["data:,"] + + def parse(self, response): + raise CloseSpider("foo") + + close_reasons = [] + + def track_close_reason(spider, reason): + close_reasons.append(reason) + + crawler = get_crawler(TestSpider) + crawler.signals.connect(track_close_reason, signal=signals.spider_closed) + await maybe_deferred_to_future(crawler.crawl()) + assert len(close_reasons) == 1 + assert close_reasons[0] == "foo"