From d27d0a4ed9f9dc52aa8e9696fd247f78e9b1b2e2 Mon Sep 17 00:00:00 2001 From: NavpreetSSidhu <116211004+NavpreetSSidhu@users.noreply.github.com> Date: Fri, 8 Aug 2025 00:56:10 +0530 Subject: [PATCH] Deprecate setting overrides through spider attributes (#6994) --- scrapy/core/downloader/__init__.py | 14 ++++++++++++++ scrapy/downloadermiddlewares/httpcompression.py | 17 ++++++++++++++++- ...test_downloadermiddleware_httpcompression.py | 8 ++++++++ tests/test_spider.py | 2 ++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 627e16331..d368d7f92 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -93,9 +93,23 @@ def _get_concurrency_delay( ) -> tuple[int, float]: delay: float = settings.getfloat("DOWNLOAD_DELAY") if hasattr(spider, "download_delay"): + warnings.warn( + "The 'download_delay' spider attribute is deprecated. " + "Use Spider.custom_settings or Spider.update_settings() instead. " + "The corresponding setting name is 'DOWNLOAD_DELAY'.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) delay = spider.download_delay if hasattr(spider, "max_concurrent_requests"): + warnings.warn( + "The 'max_concurrent_requests' spider attribute is deprecated. " + "Use Spider.custom_settings or Spider.update_settings() instead. " + "The corresponding setting name is 'CONCURRENT_REQUESTS'.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) concurrency = spider.max_concurrent_requests return concurrency, delay diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 58891b952..314baf1dc 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,11 +1,12 @@ from __future__ import annotations +import warnings from itertools import chain from logging import getLogger from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals -from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.utils._compression import ( @@ -74,8 +75,22 @@ class HttpCompressionMiddleware: def open_spider(self, spider: Spider) -> None: if hasattr(spider, "download_maxsize"): + warnings.warn( + "The 'download_maxsize' spider attribute is deprecated. " + "Use Spider.custom_settings or Spider.update_settings() instead. " + "The corresponding setting name is 'DOWNLOAD_MAXSIZE'.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) self._max_size = spider.download_maxsize if hasattr(spider, "download_warnsize"): + warnings.warn( + "The 'download_warnsize' spider attribute is deprecated. " + "Use Spider.custom_settings or Spider.update_settings() instead. " + "The corresponding setting name is 'DOWNLOAD_WARNSIZE'.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) self._warn_size = spider.download_warnsize def process_request( diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 0b3941c09..4389f044c 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -551,17 +551,21 @@ class TestHttpCompression: with pytest.raises(IgnoreRequest): mw.process_response(response.request, response, spider) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr_br(self): _skip_if_no_br() self._test_compression_bomb_spider_attr("br") + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr_deflate(self): self._test_compression_bomb_spider_attr("deflate") + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr_gzip(self): self._test_compression_bomb_spider_attr("gzip") + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr_zstd(self): _skip_if_no_zstd() @@ -664,17 +668,21 @@ class TestHttpCompression: ), ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_download_warnsize_spider_attr_br(self): _skip_if_no_br() self._test_download_warnsize_spider_attr("br") + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_download_warnsize_spider_attr_deflate(self): self._test_download_warnsize_spider_attr("deflate") + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_download_warnsize_spider_attr_gzip(self): self._test_download_warnsize_spider_attr("gzip") + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_download_warnsize_spider_attr_zstd(self): _skip_if_no_zstd() diff --git a/tests/test_spider.py b/tests/test_spider.py index 4a532ed78..dcc9d1e82 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -726,6 +726,7 @@ Sitemap: /sitemap-relative-url.xml response = Response(url="https://example.com", body=body, request=request) assert spider._get_sitemap_body(response) is None + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr(self): class DownloadMaxSizeSpider(self.spider_class): download_maxsize = 10_000_000 @@ -773,6 +774,7 @@ Sitemap: /sitemap-relative-url.xml ), ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_download_warnsize_spider_attr(self): class DownloadWarnSizeSpider(self.spider_class): download_warnsize = 10_000_000