Merge remote-tracking branch 'origin/master' into change-init-order

docs/topics/api.rst

@@ -100,7 +100,7 @@ how you :ref:`configure the downloader middlewares
 
         Starts the crawler by instantiating its spider class with the given
         ``args`` and ``kwargs`` arguments, while setting the execution engine in
-        motion.
+        motion. Should be called only once.
 
         Returns a deferred that is fired when the crawl is finished.
 

scrapy/crawler.py

@@ -107,6 +107,7 @@ class Crawler:
         self._init_reactor = init_reactor
 
         self.crawling: bool = False
+        self._started: bool = False
         self.spider: Optional[Spider] = None
         self.engine: Optional[ExecutionEngine] = None
 
@@ -114,7 +115,13 @@ class Crawler:
     def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
         if self.crawling:
             raise RuntimeError("Crawling already taking place")
-        self.crawling = True
+        if self._started:
+            warnings.warn(
+                "Running Crawler.crawl() more than once is deprecated.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+        self.crawling = self._started = True
 
         try:
             self.spider = self._create_spider(*args, **kwargs)

tests/test_crawler.py

@@ -6,6 +6,7 @@ import sys
 import warnings
 from pathlib import Path
 
+import pytest
 from packaging.version import parse as parse_version
 from pytest import mark, raises
 from twisted.internet import defer
@@ -46,6 +47,16 @@ class CrawlerTestCase(BaseCrawlerTest):
         with raises(ValueError):
             Crawler(DefaultSpider())
 
+    @defer.inlineCallbacks
+    def test_crawler_crawl_twice_deprecated(self):
+        crawler = Crawler(NoRequestsSpider)
+        yield crawler.crawl()
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match=r"Running Crawler.crawl\(\) more than once is deprecated",
+        ):
+            yield crawler.crawl()
+
 
 class CrawlerLoggingTestCase(unittest.TestCase):
     def test_no_root_handler_installed(self):