Deprecate setting overrides through spider attributes (#6994)

scrapy/core/downloader/__init__.py

@@ -93,9 +93,23 @@ def _get_concurrency_delay(
 ) -> tuple[int, float]:
     delay: float = settings.getfloat("DOWNLOAD_DELAY")
     if hasattr(spider, "download_delay"):
+        warnings.warn(
+            "The 'download_delay' spider attribute is deprecated. "
+            "Use Spider.custom_settings or Spider.update_settings() instead. "
+            "The corresponding setting name is 'DOWNLOAD_DELAY'.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         delay = spider.download_delay
 
     if hasattr(spider, "max_concurrent_requests"):
+        warnings.warn(
+            "The 'max_concurrent_requests' spider attribute is deprecated. "
+            "Use Spider.custom_settings or Spider.update_settings() instead. "
+            "The corresponding setting name is 'CONCURRENT_REQUESTS'.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         concurrency = spider.max_concurrent_requests
 
     return concurrency, delay

scrapy/downloadermiddlewares/httpcompression.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
+import warnings
 from itertools import chain
 from logging import getLogger
 from typing import TYPE_CHECKING, Any
 
 from scrapy import Request, Spider, signals
-from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.utils._compression import (
@@ -74,8 +75,22 @@ class HttpCompressionMiddleware:
 
     def open_spider(self, spider: Spider) -> None:
         if hasattr(spider, "download_maxsize"):
+            warnings.warn(
+                "The 'download_maxsize' spider attribute is deprecated. "
+                "Use Spider.custom_settings or Spider.update_settings() instead. "
+                "The corresponding setting name is 'DOWNLOAD_MAXSIZE'.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
             self._max_size = spider.download_maxsize
         if hasattr(spider, "download_warnsize"):
+            warnings.warn(
+                "The 'download_warnsize' spider attribute is deprecated. "
+                "Use Spider.custom_settings or Spider.update_settings() instead. "
+                "The corresponding setting name is 'DOWNLOAD_WARNSIZE'.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
             self._warn_size = spider.download_warnsize
 
     def process_request(

tests/test_downloadermiddleware_httpcompression.py

@@ -551,17 +551,21 @@ class TestHttpCompression:
         with pytest.raises(IgnoreRequest):
             mw.process_response(response.request, response, spider)
 
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_compression_bomb_spider_attr_br(self):
         _skip_if_no_br()
 
         self._test_compression_bomb_spider_attr("br")
 
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_compression_bomb_spider_attr_deflate(self):
         self._test_compression_bomb_spider_attr("deflate")
 
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_compression_bomb_spider_attr_gzip(self):
         self._test_compression_bomb_spider_attr("gzip")
 
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_compression_bomb_spider_attr_zstd(self):
         _skip_if_no_zstd()
 
@@ -664,17 +668,21 @@ class TestHttpCompression:
             ),
         )
 
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_download_warnsize_spider_attr_br(self):
         _skip_if_no_br()
 
         self._test_download_warnsize_spider_attr("br")
 
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_download_warnsize_spider_attr_deflate(self):
         self._test_download_warnsize_spider_attr("deflate")
 
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_download_warnsize_spider_attr_gzip(self):
         self._test_download_warnsize_spider_attr("gzip")
 
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_download_warnsize_spider_attr_zstd(self):
         _skip_if_no_zstd()
 

tests/test_spider.py

@@ -726,6 +726,7 @@ Sitemap: /sitemap-relative-url.xml
         response = Response(url="https://example.com", body=body, request=request)
         assert spider._get_sitemap_body(response) is None
 
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_compression_bomb_spider_attr(self):
         class DownloadMaxSizeSpider(self.spider_class):
             download_maxsize = 10_000_000
@@ -773,6 +774,7 @@ Sitemap: /sitemap-relative-url.xml
             ),
         )
 
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_download_warnsize_spider_attr(self):
         class DownloadWarnSizeSpider(self.spider_class):
             download_warnsize = 10_000_000