diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 11251a92c..35e3aaa24 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -546,7 +546,7 @@ performed by the Scrapy downloader. CONCURRENT_REQUESTS_PER_DOMAIN ------------------------------ -Default: ``1`` (:ref:`fallback `: ``8``) +Default: ``1`` The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single domain. @@ -899,7 +899,7 @@ Whether to enable downloader stats collection. DOWNLOAD_DELAY -------------- -Default: ``1`` (:ref:`fallback `: ``0``) +Default: ``1`` Minimum seconds to wait between 2 consecutive requests to the same domain. @@ -1786,17 +1786,11 @@ Adjust redirect request priority relative to original request: ROBOTSTXT_OBEY -------------- -Default: ``True`` (:ref:`fallback `: ``False``) +Default: ``True`` If enabled, Scrapy will respect robots.txt policies. For more information see :ref:`topics-dlmw-robots`. -.. note:: - - While the default value is ``False`` for historical reasons, - this option is enabled by default in settings.py file generated - by ``scrapy startproject`` command. - .. setting:: ROBOTSTXT_PARSER ROBOTSTXT_PARSER diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c72752915..c4e1bf297 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -96,6 +96,7 @@ class Crawler: return self.addons.load_settings(self.settings) + self._warn_on_deprecated_default_settings() self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) @@ -151,6 +152,23 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + def _warn_on_deprecated_default_settings(self) -> None: + default_priority = SETTINGS_PRIORITIES["default"] + for setting_name, current_default, future_default in ( + ("CONCURRENT_REQUESTS_PER_DOMAIN", 8, 1), + ("DOWNLOAD_DELAY", 0, 1), + ("ROBOTSTXT_OBEY", False, True), + ): + if self.settings.getpriority(setting_name) == default_priority: + warnings.warn( + f"The default value of {setting_name} will change from " + f"{current_default!r} to {future_default!r} in a future " + f"Scrapy version. Explicitly set {setting_name} in your " + f"settings to silence this warning.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 90ed70262..8d6366e04 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -66,13 +66,15 @@ def get_crawler( will be used to populate the crawler settings with a project level priority. """ - # When needed, useful settings can be added here, e.g. ones that prevent - # deprecation warnings. settings: dict[str, Any] = { "TELNETCONSOLE_ENABLED": False, **get_reactor_settings(), **(settings_dict or {}), } + if prevent_warnings: + settings.setdefault("CONCURRENT_REQUESTS_PER_DOMAIN", 8) + settings.setdefault("DOWNLOAD_DELAY", 0) + settings.setdefault("ROBOTSTXT_OBEY", False) runner: CrawlerRunnerBase if is_reactor_installed(): runner = CrawlerRunner(settings) diff --git a/tests/AsyncCrawlerProcess/reactorless_datauri.py b/tests/AsyncCrawlerProcess/reactorless_datauri.py index 5270b7b83..ad1fc9e38 100644 --- a/tests/AsyncCrawlerProcess/reactorless_datauri.py +++ b/tests/AsyncCrawlerProcess/reactorless_datauri.py @@ -12,7 +12,14 @@ class DataSpider(Spider): return {"data": response.text} -process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "CONCURRENT_REQUESTS_PER_DOMAIN": 8, + "DOWNLOAD_DELAY": 0, + "ROBOTSTXT_OBEY": False, + } +) process.crawl(DataSpider) process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_simple.py b/tests/AsyncCrawlerProcess/reactorless_simple.py index 33b4e8cb2..cd7fffa38 100644 --- a/tests/AsyncCrawlerProcess/reactorless_simple.py +++ b/tests/AsyncCrawlerProcess/reactorless_simple.py @@ -12,7 +12,14 @@ class NoRequestsSpider(scrapy.Spider): yield -process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "CONCURRENT_REQUESTS_PER_DOMAIN": 8, + "DOWNLOAD_DELAY": 0, + "ROBOTSTXT_OBEY": False, + } +) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/AsyncCrawlerRunner/reactorless_datauri.py b/tests/AsyncCrawlerRunner/reactorless_datauri.py index d964f9c1c..5afa3ef87 100644 --- a/tests/AsyncCrawlerRunner/reactorless_datauri.py +++ b/tests/AsyncCrawlerRunner/reactorless_datauri.py @@ -17,7 +17,14 @@ class DataSpider(Spider): async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) + runner = AsyncCrawlerRunner( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "CONCURRENT_REQUESTS_PER_DOMAIN": 8, + "DOWNLOAD_DELAY": 0, + "ROBOTSTXT_OBEY": False, + } + ) await runner.crawl(DataSpider) diff --git a/tests/AsyncCrawlerRunner/reactorless_simple.py b/tests/AsyncCrawlerRunner/reactorless_simple.py index e91c7c89b..0e3684d32 100644 --- a/tests/AsyncCrawlerRunner/reactorless_simple.py +++ b/tests/AsyncCrawlerRunner/reactorless_simple.py @@ -17,7 +17,14 @@ class NoRequestsSpider(Spider): async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) + runner = AsyncCrawlerRunner( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "CONCURRENT_REQUESTS_PER_DOMAIN": 8, + "DOWNLOAD_DELAY": 0, + "ROBOTSTXT_OBEY": False, + } + ) await runner.crawl(NoRequestsSpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0cddfd0ed..44911dc45 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import logging import re +import warnings from pathlib import Path from typing import Any, ClassVar @@ -81,6 +82,62 @@ class TestCrawler(TestBaseCrawler): assert not settings.frozen assert crawler.settings.frozen + @pytest.mark.parametrize( + ("setting_name", "current_default", "future_default"), + [ + ("CONCURRENT_REQUESTS_PER_DOMAIN", 8, 1), + ("DOWNLOAD_DELAY", 0, 1), + ("ROBOTSTXT_OBEY", False, True), + ], + ) + def test_default_value_deprecation_warning( + self, setting_name: str, current_default: Any, future_default: Any + ) -> None: + crawler = get_raw_crawler() + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + crawler._apply_settings() + messages = [str(warning.message) for warning in w] + assert any( + setting_name in msg + and repr(current_default) in msg + and repr(future_default) in msg + for msg in messages + ) + + @pytest.mark.parametrize( + "setting_name", + ["CONCURRENT_REQUESTS_PER_DOMAIN", "DOWNLOAD_DELAY", "ROBOTSTXT_OBEY"], + ) + def test_no_deprecation_warning_when_set(self, setting_name: str) -> None: + crawler = get_raw_crawler(settings_dict={setting_name: 1}) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + crawler._apply_settings() + assert not any( + setting_name in str(warning.message) + and issubclass(warning.category, ScrapyDeprecationWarning) + for warning in w + ) + + def test_get_crawler_prevent_warnings_false(self) -> None: + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + get_crawler(prevent_warnings=False) + setting_names = { + "CONCURRENT_REQUESTS_PER_DOMAIN", + "DOWNLOAD_DELAY", + "ROBOTSTXT_OBEY", + } + warned_messages = [ + str(warning.message) + for warning in w + if issubclass(warning.category, ScrapyDeprecationWarning) + ] + assert all( + any(name in msg for msg in warned_messages) for name in setting_names + ) + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar"