From 0ceb6509f078404af731a7d35612fa492b4e2331 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 15 Jun 2026 13:31:07 +0200 Subject: [PATCH 1/3] Deprecate some current default values --- docs/topics/settings.rst | 12 +++--------- scrapy/crawler.py | 18 ++++++++++++++++++ tests/test_crawler.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 455985a56..2f2310820 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -519,7 +519,7 @@ performed by the Scrapy downloader. CONCURRENT_REQUESTS_PER_DOMAIN ------------------------------ -Default: ``1`` (:ref:`fallback `: ``8``) +Default: ``1`` The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single domain. @@ -855,7 +855,7 @@ Whether to enable downloader stats collection. DOWNLOAD_DELAY -------------- -Default: ``1`` (:ref:`fallback `: ``0``) +Default: ``1`` Minimum seconds to wait between 2 consecutive requests to the same domain. @@ -1714,17 +1714,11 @@ Adjust redirect request priority relative to original request: ROBOTSTXT_OBEY -------------- -Default: ``True`` (:ref:`fallback `: ``False``) +Default: ``True`` If enabled, Scrapy will respect robots.txt policies. For more information see :ref:`topics-dlmw-robots`. -.. note:: - - While the default value is ``False`` for historical reasons, - this option is enabled by default in settings.py file generated - by ``scrapy startproject`` command. - .. setting:: ROBOTSTXT_PARSER ROBOTSTXT_PARSER diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 8e3ae7879..525ff083c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -96,6 +96,7 @@ class Crawler: return self.addons.load_settings(self.settings) + self._warn_on_deprecated_default_settings() self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) @@ -151,6 +152,23 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + def _warn_on_deprecated_default_settings(self) -> None: + default_priority = SETTINGS_PRIORITIES["default"] + for setting_name, current_default, future_default in ( + ("CONCURRENT_REQUESTS_PER_DOMAIN", 8, 1), + ("DOWNLOAD_DELAY", 0, 1), + ("ROBOTSTXT_OBEY", False, True), + ): + if self.settings.getpriority(setting_name) == default_priority: + warnings.warn( + f"The default value of {setting_name} will change from " + f"{current_default!r} to {future_default!r} in a future " + f"Scrapy version. Explicitly set {setting_name} in your " + f"settings to silence this warning.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 31d195c4b..f45f495b2 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -82,6 +82,44 @@ class TestCrawler(TestBaseCrawler): assert not settings.frozen assert crawler.settings.frozen + @pytest.mark.parametrize( + ("setting_name", "current_default", "future_default"), + [ + ("CONCURRENT_REQUESTS_PER_DOMAIN", 8, 1), + ("DOWNLOAD_DELAY", 0, 1), + ("ROBOTSTXT_OBEY", False, True), + ], + ) + def test_default_value_deprecation_warning( + self, setting_name: str, current_default: Any, future_default: Any + ) -> None: + crawler = get_raw_crawler() + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + crawler._apply_settings() + messages = [str(warning.message) for warning in w] + assert any( + setting_name in msg + and repr(current_default) in msg + and repr(future_default) in msg + for msg in messages + ) + + @pytest.mark.parametrize( + "setting_name", + ["CONCURRENT_REQUESTS_PER_DOMAIN", "DOWNLOAD_DELAY", "ROBOTSTXT_OBEY"], + ) + def test_no_deprecation_warning_when_set(self, setting_name: str) -> None: + crawler = get_raw_crawler(settings_dict={setting_name: 1}) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + crawler._apply_settings() + assert not any( + setting_name in str(warning.message) + and issubclass(warning.category, ScrapyDeprecationWarning) + for warning in w + ) + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar" From 18507ca5d567ba3e1c4681e2246bb2cc6afd8945 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 22 Jun 2026 11:34:08 +0200 Subject: [PATCH 2/3] Solve warning-related issues --- scrapy/utils/test.py | 6 ++++-- tests/AsyncCrawlerProcess/reactorless_datauri.py | 9 ++++++++- tests/AsyncCrawlerProcess/reactorless_simple.py | 9 ++++++++- tests/AsyncCrawlerRunner/reactorless_datauri.py | 9 ++++++++- tests/AsyncCrawlerRunner/reactorless_simple.py | 9 ++++++++- tests/test_crawler.py | 1 + 6 files changed, 37 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index b4e20c3c6..90eed348e 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -66,12 +66,14 @@ def get_crawler( will be used to populate the crawler settings with a project level priority. """ - # When needed, useful settings can be added here, e.g. ones that prevent - # deprecation warnings. settings: dict[str, Any] = { **get_reactor_settings(), **(settings_dict or {}), } + if prevent_warnings: + settings.setdefault("CONCURRENT_REQUESTS_PER_DOMAIN", 8) + settings.setdefault("DOWNLOAD_DELAY", 0) + settings.setdefault("ROBOTSTXT_OBEY", False) runner: CrawlerRunnerBase if is_reactor_installed(): runner = CrawlerRunner(settings) diff --git a/tests/AsyncCrawlerProcess/reactorless_datauri.py b/tests/AsyncCrawlerProcess/reactorless_datauri.py index 5270b7b83..ad1fc9e38 100644 --- a/tests/AsyncCrawlerProcess/reactorless_datauri.py +++ b/tests/AsyncCrawlerProcess/reactorless_datauri.py @@ -12,7 +12,14 @@ class DataSpider(Spider): return {"data": response.text} -process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "CONCURRENT_REQUESTS_PER_DOMAIN": 8, + "DOWNLOAD_DELAY": 0, + "ROBOTSTXT_OBEY": False, + } +) process.crawl(DataSpider) process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_simple.py b/tests/AsyncCrawlerProcess/reactorless_simple.py index 33b4e8cb2..cd7fffa38 100644 --- a/tests/AsyncCrawlerProcess/reactorless_simple.py +++ b/tests/AsyncCrawlerProcess/reactorless_simple.py @@ -12,7 +12,14 @@ class NoRequestsSpider(scrapy.Spider): yield -process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "CONCURRENT_REQUESTS_PER_DOMAIN": 8, + "DOWNLOAD_DELAY": 0, + "ROBOTSTXT_OBEY": False, + } +) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/AsyncCrawlerRunner/reactorless_datauri.py b/tests/AsyncCrawlerRunner/reactorless_datauri.py index d964f9c1c..5afa3ef87 100644 --- a/tests/AsyncCrawlerRunner/reactorless_datauri.py +++ b/tests/AsyncCrawlerRunner/reactorless_datauri.py @@ -17,7 +17,14 @@ class DataSpider(Spider): async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) + runner = AsyncCrawlerRunner( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "CONCURRENT_REQUESTS_PER_DOMAIN": 8, + "DOWNLOAD_DELAY": 0, + "ROBOTSTXT_OBEY": False, + } + ) await runner.crawl(DataSpider) diff --git a/tests/AsyncCrawlerRunner/reactorless_simple.py b/tests/AsyncCrawlerRunner/reactorless_simple.py index e91c7c89b..0e3684d32 100644 --- a/tests/AsyncCrawlerRunner/reactorless_simple.py +++ b/tests/AsyncCrawlerRunner/reactorless_simple.py @@ -17,7 +17,14 @@ class NoRequestsSpider(Spider): async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) + runner = AsyncCrawlerRunner( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "CONCURRENT_REQUESTS_PER_DOMAIN": 8, + "DOWNLOAD_DELAY": 0, + "ROBOTSTXT_OBEY": False, + } + ) await runner.crawl(NoRequestsSpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index a3235003e..f67f23778 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import logging import re +import warnings from pathlib import Path from typing import Any, ClassVar From 9f335d379880a357f98be4d5645a3cb230227fd3 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 22 Jun 2026 12:24:07 +0200 Subject: [PATCH 3/3] Add a test to cover the default warnings from get_crawler() --- tests/test_crawler.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f67f23778..462db656d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -120,6 +120,24 @@ class TestCrawler(TestBaseCrawler): for warning in w ) + def test_get_crawler_prevent_warnings_false(self) -> None: + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + get_crawler(prevent_warnings=False) + setting_names = { + "CONCURRENT_REQUESTS_PER_DOMAIN", + "DOWNLOAD_DELAY", + "ROBOTSTXT_OBEY", + } + warned_messages = [ + str(warning.message) + for warning in w + if issubclass(warning.category, ScrapyDeprecationWarning) + ] + assert all( + any(name in msg for msg in warned_messages) for name in setting_names + ) + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar"