diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 16c28405c..175c877de 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -100,7 +100,7 @@ how you :ref:`configure the downloader middlewares Starts the crawler by instantiating its spider class with the given ``args`` and ``kwargs`` arguments, while setting the execution engine in - motion. + motion. Should be called only once. Returns a deferred that is fired when the crawl is finished. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 44ffc44ce..974754964 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -107,6 +107,7 @@ class Crawler: self._init_reactor = init_reactor self.crawling: bool = False + self._started: bool = False self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None @@ -114,7 +115,13 @@ class Crawler: def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: if self.crawling: raise RuntimeError("Crawling already taking place") - self.crawling = True + if self._started: + warnings.warn( + "Running Crawler.crawl() more than once is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self.crawling = self._started = True try: self.spider = self._create_spider(*args, **kwargs) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 1fe865ee4..120991ae7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -6,6 +6,7 @@ import sys import warnings from pathlib import Path +import pytest from packaging.version import parse as parse_version from pytest import mark, raises from twisted.internet import defer @@ -46,6 +47,16 @@ class CrawlerTestCase(BaseCrawlerTest): with raises(ValueError): Crawler(DefaultSpider()) + @defer.inlineCallbacks + def test_crawler_crawl_twice_deprecated(self): + crawler = Crawler(NoRequestsSpider) + yield crawler.crawl() + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Running Crawler.crawl\(\) more than once is deprecated", + ): + yield crawler.crawl() + class CrawlerLoggingTestCase(unittest.TestCase): def test_no_root_handler_installed(self):