Deprecate some current default values

docs/topics/settings.rst

@@ -519,7 +519,7 @@ performed by the Scrapy downloader.
 CONCURRENT_REQUESTS_PER_DOMAIN
 ------------------------------
 
-Default: ``1`` (:ref:`fallback <default-settings>`: ``8``)
+Default: ``1``
 
 The maximum number of concurrent (i.e. simultaneous) requests that will be
 performed to any single domain.
@@ -855,7 +855,7 @@ Whether to enable downloader stats collection.
 DOWNLOAD_DELAY
 --------------
 
-Default: ``1`` (:ref:`fallback <default-settings>`: ``0``)
+Default: ``1``
 
 Minimum seconds to wait between 2 consecutive requests to the same domain.
 
@@ -1714,17 +1714,11 @@ Adjust redirect request priority relative to original request:
 ROBOTSTXT_OBEY
 --------------
 
-Default: ``True`` (:ref:`fallback <default-settings>`: ``False``)
+Default: ``True``
 
 If enabled, Scrapy will respect robots.txt policies. For more information see
 :ref:`topics-dlmw-robots`.
 
-.. note::
-
-    While the default value is ``False`` for historical reasons,
-    this option is enabled by default in settings.py file generated
-    by ``scrapy startproject`` command.
-
 .. setting:: ROBOTSTXT_PARSER
 
 ROBOTSTXT_PARSER

scrapy/crawler.py

@@ -96,6 +96,7 @@ class Crawler:
             return
 
         self.addons.load_settings(self.settings)
+        self._warn_on_deprecated_default_settings()
         self.stats = load_object(self.settings["STATS_CLASS"])(self)
 
         lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
@@ -151,6 +152,23 @@ class Crawler:
             "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
         )
 
+    def _warn_on_deprecated_default_settings(self) -> None:
+        default_priority = SETTINGS_PRIORITIES["default"]
+        for setting_name, current_default, future_default in (
+            ("CONCURRENT_REQUESTS_PER_DOMAIN", 8, 1),
+            ("DOWNLOAD_DELAY", 0, 1),
+            ("ROBOTSTXT_OBEY", False, True),
+        ):
+            if self.settings.getpriority(setting_name) == default_priority:
+                warnings.warn(
+                    f"The default value of {setting_name} will change from "
+                    f"{current_default!r} to {future_default!r} in a future "
+                    f"Scrapy version. Explicitly set {setting_name} in your "
+                    f"settings to silence this warning.",
+                    category=ScrapyDeprecationWarning,
+                    stacklevel=3,
+                )
+
     def _apply_reactorless_default_settings(self) -> None:
         """Change some setting defaults when not using a Twisted reactor.
 

tests/test_crawler.py

@@ -82,6 +82,44 @@ class TestCrawler(TestBaseCrawler):
         assert not settings.frozen
         assert crawler.settings.frozen
 
+    @pytest.mark.parametrize(
+        ("setting_name", "current_default", "future_default"),
+        [
+            ("CONCURRENT_REQUESTS_PER_DOMAIN", 8, 1),
+            ("DOWNLOAD_DELAY", 0, 1),
+            ("ROBOTSTXT_OBEY", False, True),
+        ],
+    )
+    def test_default_value_deprecation_warning(
+        self, setting_name: str, current_default: Any, future_default: Any
+    ) -> None:
+        crawler = get_raw_crawler()
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            crawler._apply_settings()
+        messages = [str(warning.message) for warning in w]
+        assert any(
+            setting_name in msg
+            and repr(current_default) in msg
+            and repr(future_default) in msg
+            for msg in messages
+        )
+
+    @pytest.mark.parametrize(
+        "setting_name",
+        ["CONCURRENT_REQUESTS_PER_DOMAIN", "DOWNLOAD_DELAY", "ROBOTSTXT_OBEY"],
+    )
+    def test_no_deprecation_warning_when_set(self, setting_name: str) -> None:
+        crawler = get_raw_crawler(settings_dict={setting_name: 1})
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            crawler._apply_settings()
+        assert not any(
+            setting_name in str(warning.message)
+            and issubclass(warning.category, ScrapyDeprecationWarning)
+            for warning in w
+        )
+
     def test_crawler_accepts_dict(self) -> None:
         crawler = get_crawler(DefaultSpider, {"foo": "bar"})
         assert crawler.settings["foo"] == "bar"