diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 455985a56..2f2310820 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -519,7 +519,7 @@ performed by the Scrapy downloader. CONCURRENT_REQUESTS_PER_DOMAIN ------------------------------ -Default: ``1`` (:ref:`fallback `: ``8``) +Default: ``1`` The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single domain. @@ -855,7 +855,7 @@ Whether to enable downloader stats collection. DOWNLOAD_DELAY -------------- -Default: ``1`` (:ref:`fallback `: ``0``) +Default: ``1`` Minimum seconds to wait between 2 consecutive requests to the same domain. @@ -1714,17 +1714,11 @@ Adjust redirect request priority relative to original request: ROBOTSTXT_OBEY -------------- -Default: ``True`` (:ref:`fallback `: ``False``) +Default: ``True`` If enabled, Scrapy will respect robots.txt policies. For more information see :ref:`topics-dlmw-robots`. -.. note:: - - While the default value is ``False`` for historical reasons, - this option is enabled by default in settings.py file generated - by ``scrapy startproject`` command. - .. setting:: ROBOTSTXT_PARSER ROBOTSTXT_PARSER diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 8e3ae7879..525ff083c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -96,6 +96,7 @@ class Crawler: return self.addons.load_settings(self.settings) + self._warn_on_deprecated_default_settings() self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) @@ -151,6 +152,23 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + def _warn_on_deprecated_default_settings(self) -> None: + default_priority = SETTINGS_PRIORITIES["default"] + for setting_name, current_default, future_default in ( + ("CONCURRENT_REQUESTS_PER_DOMAIN", 8, 1), + ("DOWNLOAD_DELAY", 0, 1), + ("ROBOTSTXT_OBEY", False, True), + ): + if self.settings.getpriority(setting_name) == default_priority: + warnings.warn( + f"The default value of {setting_name} will change from " + f"{current_default!r} to {future_default!r} in a future " + f"Scrapy version. Explicitly set {setting_name} in your " + f"settings to silence this warning.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 31d195c4b..f45f495b2 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -82,6 +82,44 @@ class TestCrawler(TestBaseCrawler): assert not settings.frozen assert crawler.settings.frozen + @pytest.mark.parametrize( + ("setting_name", "current_default", "future_default"), + [ + ("CONCURRENT_REQUESTS_PER_DOMAIN", 8, 1), + ("DOWNLOAD_DELAY", 0, 1), + ("ROBOTSTXT_OBEY", False, True), + ], + ) + def test_default_value_deprecation_warning( + self, setting_name: str, current_default: Any, future_default: Any + ) -> None: + crawler = get_raw_crawler() + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + crawler._apply_settings() + messages = [str(warning.message) for warning in w] + assert any( + setting_name in msg + and repr(current_default) in msg + and repr(future_default) in msg + for msg in messages + ) + + @pytest.mark.parametrize( + "setting_name", + ["CONCURRENT_REQUESTS_PER_DOMAIN", "DOWNLOAD_DELAY", "ROBOTSTXT_OBEY"], + ) + def test_no_deprecation_warning_when_set(self, setting_name: str) -> None: + crawler = get_raw_crawler(settings_dict={setting_name: 1}) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + crawler._apply_settings() + assert not any( + setting_name in str(warning.message) + and issubclass(warning.category, ScrapyDeprecationWarning) + for warning in w + ) + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar"