Merge pull request #6040 from wRAR/deprecate-double-crawl

Deprecate running Crawler.crawl() twice.

docs/topics/api.rst

@@ -100,7 +100,7 @@ how you :ref:`configure the downloader middlewares
 
         Starts the crawler by instantiating its spider class with the given
         ``args`` and ``kwargs`` arguments, while setting the execution engine in
-        motion.
+        motion. Should be called only once.
 
         Returns a deferred that is fired when the crawl is finished.
 

scrapy/crawler.py

@@ -124,6 +124,7 @@ class Crawler:
 
         self.settings.freeze()
         self.crawling: bool = False
+        self._started: bool = False
         self.spider: Optional[Spider] = None
         self.engine: Optional[ExecutionEngine] = None
 
@@ -131,7 +132,13 @@ class Crawler:
     def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
         if self.crawling:
             raise RuntimeError("Crawling already taking place")
-        self.crawling = True
+        if self._started:
+            warnings.warn(
+                "Running Crawler.crawl() more than once is deprecated.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+        self.crawling = self._started = True
 
         try:
             self.spider = self._create_spider(*args, **kwargs)

tests/test_crawl.py

@@ -123,7 +123,9 @@ class CrawlTestCase(TestCase):
         self.assertTrue(crawler.spider.t2 == 0)
         self.assertTrue(crawler.spider.t2_err > 0)
         self.assertTrue(crawler.spider.t2_err > crawler.spider.t1)
+
         # server hangs after receiving response headers
+        crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
         yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 == 0)
@@ -201,6 +203,7 @@ class CrawlTestCase(TestCase):
         )
         self.assertEqual(crawler.spider.visited, 6)
 
+        crawler = get_crawler(DuplicateStartRequestsSpider, settings)
         yield crawler.crawl(
             dont_filter=False,
             distinct_urls=3,

tests/test_crawler.py

@@ -6,6 +6,7 @@ import sys
 import warnings
 from pathlib import Path
 
+import pytest
 from packaging.version import parse as parse_version
 from pytest import mark, raises
 from twisted.internet import defer
@@ -67,6 +68,16 @@ class CrawlerTestCase(BaseCrawlerTest):
         with raises(ValueError):
             Crawler(DefaultSpider())
 
+    @defer.inlineCallbacks
+    def test_crawler_crawl_twice_deprecated(self):
+        crawler = Crawler(NoRequestsSpider)
+        yield crawler.crawl()
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match=r"Running Crawler.crawl\(\) more than once is deprecated",
+        ):
+            yield crawler.crawl()
+
 
 class SpiderSettingsTestCase(unittest.TestCase):
     def test_spider_custom_settings(self):

tests/test_downloader_handlers.py

@@ -743,6 +743,7 @@ class Http11MockServerTestCase(unittest.TestCase):
 
         # See issue https://twistedmatrix.com/trac/ticket/8175
         raise unittest.SkipTest("xpayload fails on PY3")
+        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
         request.headers.setdefault(b"Accept-Encoding", b"gzip,deflate")
         request = request.replace(url=self.mockserver.url("/xpayload"))
         yield crawler.crawl(seed=request)