diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index d29dc2c06..455326a0a 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -318,6 +318,13 @@ class Scraper: callback = result.request.callback or self.crawler.spider._parse warn_on_generator_with_return_value(self.crawler.spider, callback) output = callback(result, **result.request.cb_kwargs) + if isinstance(output, Deferred): + warnings.warn( + f"{callback} returned a Deferred." + f" Returning Deferreds from spider callbacks is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) else: # result is a Failure # TODO: properly type adding this attribute to a Failure result.request = request # type: ignore[attr-defined] @@ -329,6 +336,13 @@ class Scraper: output.raiseException() # else the errback returned actual output (like a callback), # which needs to be passed to iterate_spider_output() + if isinstance(output, Deferred): + warnings.warn( + f"{request.errback} returned a Deferred." + f" Returning Deferreds from spider errbacks is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) return await ensure_awaitable(iterate_spider_output(output)) @_warn_spider_arg diff --git a/tests/test_crawl.py b/tests/test_crawl.py index dc85d9d68..b0060a76a 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -9,13 +9,13 @@ from urllib.parse import urlencode, urlparse import pytest from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import inlineCallbacks, succeed from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from scrapy import Spider, signals from scrapy.crawler import CrawlerRunner -from scrapy.exceptions import CloseSpider, StopDownload +from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future @@ -720,6 +720,18 @@ class TestCrawlSpider: "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") + @inlineCallbacks + def test_spider_callback_deferred_deprecated(self): + def cb(response: Response) -> Any: + return succeed(None) + + crawler = get_crawler(SingleRequestSpider) + with pytest.warns( + ScrapyDeprecationWarning, + match="Returning Deferreds from spider callbacks is deprecated", + ): + yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) + @inlineCallbacks def test_spider_errback(self): failures = [] @@ -851,6 +863,20 @@ class TestCrawlSpider: assert "Spider error processing" not in str(log) assert "Crawled (200)" in str(log) + @inlineCallbacks + def test_spider_errback_deferred_deprecated(self): + def eb(failure: Failure) -> Any: + return succeed(None) + + crawler = get_crawler(SingleRequestSpider) + with pytest.warns( + ScrapyDeprecationWarning, + match="Returning Deferreds from spider errbacks is deprecated", + ): + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + @inlineCallbacks def test_raise_closespider(self): def cb(response):