Merge a9f86580c9 into ad43bf0c56

docs/topics/settings.rst

@@ -592,7 +592,7 @@ performed by the Scrapy downloader. Use ``0`` for no limit.
 CONCURRENT_REQUESTS_PER_DOMAIN
 ------------------------------
 
-Default: ``1`` (:ref:`fallback <default-settings>`: ``8``)
+Default: ``1``
 
 The maximum number of concurrent (i.e. simultaneous) requests that will be
 performed to any single domain.
@@ -944,7 +944,7 @@ Whether to enable downloader stats collection.
 DOWNLOAD_DELAY
 --------------
 
-Default: ``1`` (:ref:`fallback <default-settings>`: ``0``)
+Default: ``1``
 
 Minimum seconds to wait between 2 consecutive requests to the same domain.
 
@@ -1870,17 +1870,11 @@ Adjust redirect request priority relative to original request:
 ROBOTSTXT_OBEY
 --------------
 
-Default: ``True`` (:ref:`fallback <default-settings>`: ``False``)
+Default: ``True``
 
 If enabled, Scrapy will respect robots.txt policies. For more information see
 :ref:`topics-dlmw-robots`.
 
-.. note::
-
-    While the default value is ``False`` for historical reasons,
-    this option is enabled by default in settings.py file generated
-    by ``scrapy startproject`` command.
-
 .. setting:: ROBOTSTXT_PARSER
 
 ROBOTSTXT_PARSER

scrapy/crawler.py

@@ -153,6 +153,7 @@ class Crawler:
         self._apply_deprecated_spider_attr(
             "max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"
         )
+        self._warn_on_deprecated_default_settings()
         self.stats = load_object(self.settings["STATS_CLASS"])(self)
 
         lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
@@ -208,6 +209,23 @@ class Crawler:
             "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
         )
 
+    def _warn_on_deprecated_default_settings(self) -> None:
+        default_priority = SETTINGS_PRIORITIES["default"]
+        for setting_name, current_default, future_default in (
+            ("CONCURRENT_REQUESTS_PER_DOMAIN", 8, 1),
+            ("DOWNLOAD_DELAY", 0, 1),
+            ("ROBOTSTXT_OBEY", False, True),
+        ):
+            if self.settings.getpriority(setting_name) == default_priority:
+                warnings.warn(
+                    f"The default value of {setting_name} will change from "
+                    f"{current_default!r} to {future_default!r} in a future "
+                    f"Scrapy version. Explicitly set {setting_name} in your "
+                    f"settings to silence this warning.",
+                    category=ScrapyDeprecationWarning,
+                    stacklevel=3,
+                )
+
     def _apply_deprecated_spider_attr(self, attr: str, setting: str) -> None:
         """Bridge a deprecated spider attribute onto *setting*, warning about
         the deprecation (and about being ignored when *setting* is already set

scrapy/utils/test.py

@@ -66,13 +66,15 @@ def get_crawler(
     will be used to populate the crawler settings with a project level
     priority.
     """
-    # When needed, useful settings can be added here, e.g. ones that prevent
-    # deprecation warnings.
     settings: dict[str, Any] = {
         "TELNETCONSOLE_ENABLED": False,
         **get_reactor_settings(),
         **(settings_dict or {}),
     }
+    if prevent_warnings:
+        settings.setdefault("CONCURRENT_REQUESTS_PER_DOMAIN", 8)
+        settings.setdefault("DOWNLOAD_DELAY", 0)
+        settings.setdefault("ROBOTSTXT_OBEY", False)
     runner: CrawlerRunnerBase
     if is_reactor_installed():
         runner = CrawlerRunner(settings)

tests/AsyncCrawlerProcess/reactorless_datauri.py

@@ -12,7 +12,14 @@ class DataSpider(Spider):
         return {"data": response.text}
 
 
-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+process = AsyncCrawlerProcess(
+    settings={
+        "TWISTED_REACTOR_ENABLED": False,
+        "CONCURRENT_REQUESTS_PER_DOMAIN": 8,
+        "DOWNLOAD_DELAY": 0,
+        "ROBOTSTXT_OBEY": False,
+    }
+)
 
 process.crawl(DataSpider)
 process.start()

tests/AsyncCrawlerProcess/reactorless_simple.py

@@ -12,7 +12,14 @@ class NoRequestsSpider(scrapy.Spider):
         yield
 
 
-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+process = AsyncCrawlerProcess(
+    settings={
+        "TWISTED_REACTOR_ENABLED": False,
+        "CONCURRENT_REQUESTS_PER_DOMAIN": 8,
+        "DOWNLOAD_DELAY": 0,
+        "ROBOTSTXT_OBEY": False,
+    }
+)
 
 process.crawl(NoRequestsSpider)
 process.start()

tests/AsyncCrawlerRunner/reactorless_datauri.py

@@ -17,7 +17,14 @@ class DataSpider(Spider):
 
 async def main() -> None:
     configure_logging()
-    runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
+    runner = AsyncCrawlerRunner(
+        settings={
+            "TWISTED_REACTOR_ENABLED": False,
+            "CONCURRENT_REQUESTS_PER_DOMAIN": 8,
+            "DOWNLOAD_DELAY": 0,
+            "ROBOTSTXT_OBEY": False,
+        }
+    )
     await runner.crawl(DataSpider)
 
 

tests/AsyncCrawlerRunner/reactorless_simple.py

@@ -17,7 +17,14 @@ class NoRequestsSpider(Spider):
 
 async def main() -> None:
     configure_logging()
-    runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
+    runner = AsyncCrawlerRunner(
+        settings={
+            "TWISTED_REACTOR_ENABLED": False,
+            "CONCURRENT_REQUESTS_PER_DOMAIN": 8,
+            "DOWNLOAD_DELAY": 0,
+            "ROBOTSTXT_OBEY": False,
+        }
+    )
     await runner.crawl(NoRequestsSpider)
 
 

tests/test_crawler.py

@@ -5,6 +5,7 @@ import logging
 import re
 import signal
 import threading
+import warnings
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar
 from unittest.mock import MagicMock
@@ -74,6 +75,62 @@ class TestCrawler:
         assert not settings.frozen
         assert crawler.settings.frozen
 
+    @pytest.mark.parametrize(
+        ("setting_name", "current_default", "future_default"),
+        [
+            ("CONCURRENT_REQUESTS_PER_DOMAIN", 8, 1),
+            ("DOWNLOAD_DELAY", 0, 1),
+            ("ROBOTSTXT_OBEY", False, True),
+        ],
+    )
+    def test_default_value_deprecation_warning(
+        self, setting_name: str, current_default: Any, future_default: Any
+    ) -> None:
+        crawler = get_raw_crawler()
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            crawler._apply_settings()
+        messages = [str(warning.message) for warning in w]
+        assert any(
+            setting_name in msg
+            and repr(current_default) in msg
+            and repr(future_default) in msg
+            for msg in messages
+        )
+
+    @pytest.mark.parametrize(
+        "setting_name",
+        ["CONCURRENT_REQUESTS_PER_DOMAIN", "DOWNLOAD_DELAY", "ROBOTSTXT_OBEY"],
+    )
+    def test_no_deprecation_warning_when_set(self, setting_name: str) -> None:
+        crawler = get_raw_crawler(settings_dict={setting_name: 1})
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            crawler._apply_settings()
+        assert not any(
+            setting_name in str(warning.message)
+            and issubclass(warning.category, ScrapyDeprecationWarning)
+            for warning in w
+        )
+
+    def test_get_crawler_prevent_warnings_false(self) -> None:
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            get_crawler(prevent_warnings=False)
+        setting_names = {
+            "CONCURRENT_REQUESTS_PER_DOMAIN",
+            "DOWNLOAD_DELAY",
+            "ROBOTSTXT_OBEY",
+        }
+        warned_messages = [
+            str(warning.message)
+            for warning in w
+            if issubclass(warning.category, ScrapyDeprecationWarning)
+        ]
+        assert all(
+            any(name in msg for msg in warned_messages) for name in setting_names
+        )
+
     @pytest.mark.parametrize(
         "attr",
         ["extensions", "logformatter", "request_fingerprinter", "stats"],