From d3b5c9be976851deee2253a8b86b80f57a2b695e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 6 Sep 2023 19:28:08 +0400
Subject: [PATCH 1/2] Deprecate running Crawler.crawl() twice.

---
 docs/topics/api.rst               | 2 +-
 scrapy/crawler.py                 | 9 ++++++++-
 tests/test_crawl.py               | 3 +++
 tests/test_downloader_handlers.py | 1 +
 4 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 16c28405c..175c877de 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -100,7 +100,7 @@ how you :ref:`configure the downloader middlewares
 
         Starts the crawler by instantiating its spider class with the given
         ``args`` and ``kwargs`` arguments, while setting the execution engine in
-        motion.
+        motion. Should be called only once.
 
         Returns a deferred that is fired when the crawl is finished.
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 25823b6ac..3c9f28a00 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -124,6 +124,7 @@ class Crawler:
 
         self.settings.freeze()
         self.crawling: bool = False
+        self._started: bool = False
         self.spider: Optional[Spider] = None
         self.engine: Optional[ExecutionEngine] = None
 
@@ -131,7 +132,13 @@ class Crawler:
     def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
         if self.crawling:
             raise RuntimeError("Crawling already taking place")
-        self.crawling = True
+        if self._started:
+            warnings.warn(
+                "Running Crawler.crawl() more than once is deprecated.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+        self.crawling = self._started = True
 
         try:
             self.spider = self._create_spider(*args, **kwargs)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index ca9084294..96d43b2b9 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -123,7 +123,9 @@ class CrawlTestCase(TestCase):
         self.assertTrue(crawler.spider.t2 == 0)
         self.assertTrue(crawler.spider.t2_err > 0)
         self.assertTrue(crawler.spider.t2_err > crawler.spider.t1)
+
         # server hangs after receiving response headers
+        crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
         yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 == 0)
@@ -201,6 +203,7 @@ class CrawlTestCase(TestCase):
         )
         self.assertEqual(crawler.spider.visited, 6)
 
+        crawler = get_crawler(DuplicateStartRequestsSpider, settings)
         yield crawler.crawl(
             dont_filter=False,
             distinct_urls=3,
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 8459408ff..57211d97a 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -743,6 +743,7 @@ class Http11MockServerTestCase(unittest.TestCase):
 
         # See issue https://twistedmatrix.com/trac/ticket/8175
         raise unittest.SkipTest("xpayload fails on PY3")
+        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
         request.headers.setdefault(b"Accept-Encoding", b"gzip,deflate")
         request = request.replace(url=self.mockserver.url("/xpayload"))
         yield crawler.crawl(seed=request)

From b39d2d4353d95c24c56ee73c093c87268f2ef113 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 6 Sep 2023 19:44:22 +0400
Subject: [PATCH 2/2] Add a test.

---
 tests/test_crawler.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 4c5c48e6d..067934961 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -6,6 +6,7 @@ import sys
 import warnings
 from pathlib import Path
 
+import pytest
 from packaging.version import parse as parse_version
 from pytest import mark, raises
 from twisted.internet import defer
@@ -67,6 +68,16 @@ class CrawlerTestCase(BaseCrawlerTest):
         with raises(ValueError):
             Crawler(DefaultSpider())
 
+    @defer.inlineCallbacks
+    def test_crawler_crawl_twice_deprecated(self):
+        crawler = Crawler(NoRequestsSpider)
+        yield crawler.crawl()
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match=r"Running Crawler.crawl\(\) more than once is deprecated",
+        ):
+            yield crawler.crawl()
+
 
 class SpiderSettingsTestCase(unittest.TestCase):
     def test_spider_custom_settings(self):