Deprecate returning Deferreds from spider callbacks and errbacks. (#7151)

scrapy/core/scraper.py

@@ -318,6 +318,13 @@ class Scraper:
             callback = result.request.callback or self.crawler.spider._parse
             warn_on_generator_with_return_value(self.crawler.spider, callback)
             output = callback(result, **result.request.cb_kwargs)
+            if isinstance(output, Deferred):
+                warnings.warn(
+                    f"{callback} returned a Deferred."
+                    f" Returning Deferreds from spider callbacks is deprecated.",
+                    ScrapyDeprecationWarning,
+                    stacklevel=2,
+                )
         else:  # result is a Failure
             # TODO: properly type adding this attribute to a Failure
             result.request = request  # type: ignore[attr-defined]
@@ -329,6 +336,13 @@ class Scraper:
                 output.raiseException()
             # else the errback returned actual output (like a callback),
             # which needs to be passed to iterate_spider_output()
+            if isinstance(output, Deferred):
+                warnings.warn(
+                    f"{request.errback} returned a Deferred."
+                    f" Returning Deferreds from spider errbacks is deprecated.",
+                    ScrapyDeprecationWarning,
+                    stacklevel=2,
+                )
         return await ensure_awaitable(iterate_spider_output(output))
 
     @_warn_spider_arg

tests/test_crawl.py

@@ -9,13 +9,13 @@ from urllib.parse import urlencode, urlparse
 
 import pytest
 from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
+from twisted.internet.defer import inlineCallbacks, succeed
 from twisted.internet.ssl import Certificate
 from twisted.python.failure import Failure
 
 from scrapy import Spider, signals
 from scrapy.crawler import CrawlerRunner
-from scrapy.exceptions import CloseSpider, StopDownload
+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
 from scrapy.http import Request
 from scrapy.http.response import Response
 from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
@@ -720,6 +720,18 @@ class TestCrawlSpider:
             "failure"
         ].value.response.headers == crawler.spider.meta.get("headers_received")
 
+    @inlineCallbacks
+    def test_spider_callback_deferred_deprecated(self):
+        def cb(response: Response) -> Any:
+            return succeed(None)
+
+        crawler = get_crawler(SingleRequestSpider)
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Returning Deferreds from spider callbacks is deprecated",
+        ):
+            yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb)
+
     @inlineCallbacks
     def test_spider_errback(self):
         failures = []
@@ -851,6 +863,20 @@ class TestCrawlSpider:
         assert "Spider error processing" not in str(log)
         assert "Crawled (200)" in str(log)
 
+    @inlineCallbacks
+    def test_spider_errback_deferred_deprecated(self):
+        def eb(failure: Failure) -> Any:
+            return succeed(None)
+
+        crawler = get_crawler(SingleRequestSpider)
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Returning Deferreds from spider errbacks is deprecated",
+        ):
+            yield crawler.crawl(
+                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+            )
+
     @inlineCallbacks
     def test_raise_closespider(self):
         def cb(response):