Add a LOG_LEVELS setting to override built-in logger levels

docs/topics/settings.rst

@@ -377,12 +377,13 @@ is ``True``.
 Logging settings
 ----------------
 
-**Logging settings** are settings that configure the global root logging
-handler installed by :func:`~scrapy.utils.log.configure_logging`.
+**Logging settings** are settings that configure logging process-wide, mostly
+through the global root logging handler installed by
+:func:`~scrapy.utils.log.configure_logging`.
 
-These settings can be defined from a spider. However, because only 1 root
-logging handler is active per process, these settings cannot use a different
-value per spider when :ref:`running multiple spiders in the same process
+These settings can be defined from a spider. However, because logging
+configuration is global, these settings cannot use a different value per
+spider when :ref:`running multiple spiders in the same process
 <run-multiple-spiders>`.
 
 These settings are:
@@ -394,6 +395,7 @@ These settings are:
 -   :setting:`LOG_FILE_APPEND`
 -   :setting:`LOG_FORMAT`
 -   :setting:`LOG_LEVEL`
+-   :setting:`LOG_LEVELS`
 -   :setting:`LOG_SHORT_NAMES`
 -   :setting:`LOG_STDOUT`
 
@@ -1615,6 +1617,30 @@ INFO, DEBUG. For more info see :ref:`topics-logging`.
 
 .. note:: This is a :ref:`logging setting <logging-settings>`.
 
+.. setting:: LOG_LEVELS
+
+LOG_LEVELS
+----------
+
+Default: ``{}``
+
+.. versionadded:: VERSION
+
+Minimum level to log for specific loggers.
+
+It takes precedence over the levels that Scrapy sets by default::
+
+    {
+        "filelock": "ERROR",
+        "hpack": "ERROR",
+        "httpcore": "ERROR",
+        "httpx": "WARNING",
+        "scrapy": "DEBUG",
+        "twisted": "ERROR",
+    }
+
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
 .. setting:: LOG_STDOUT
 
 LOG_STDOUT

scrapy/crawler.py

@@ -22,6 +22,7 @@ from scrapy.signalmanager import SignalManager
 from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.log import (
+    _configure_logger_levels,
     configure_logging,
     get_scrapy_root_handler,
     install_scrapy_root_handler,
@@ -74,7 +75,7 @@ class Crawler:
         self.spidercls: type[Spider] = spidercls
         self.settings: Settings = settings.copy()
         self.spidercls.update_settings(self.settings)
-        self._update_root_log_handler()
+        self._update_logging()
 
         self.addons: AddonManager = AddonManager(self)
         self.signals: SignalManager = SignalManager(self)
@@ -90,9 +91,10 @@ class Crawler:
         self.spider: Spider | None = None
         self.engine: ExecutionEngine | None = None
 
-    def _update_root_log_handler(self) -> None:
+    def _update_logging(self) -> None:
         if get_scrapy_root_handler() is not None:
             # scrapy root handler already installed: update it with new settings
+            _configure_logger_levels(self.settings)
             install_scrapy_root_handler(self.settings)
 
     def _apply_settings(self) -> None:
@@ -219,7 +221,7 @@ class Crawler:
         try:
             self.spider = self._create_spider(*args, **kwargs)
             self._apply_settings()
-            self._update_root_log_handler()
+            self._update_logging()
             self.engine = self._create_engine()
             yield deferred_from_coro(self.engine.open_spider_async())
             yield deferred_from_coro(self.engine.start_async())
@@ -249,7 +251,7 @@ class Crawler:
         try:
             self.spider = self._create_spider(*args, **kwargs)
             self._apply_settings()
-            self._update_root_log_handler()
+            self._update_logging()
             self.engine = self._create_engine()
             await self.engine.open_spider_async()
             await self.engine.start_async()

scrapy/settings/default_settings.py

@@ -144,6 +144,7 @@ __all__ = [
     "LOG_FORMAT",
     "LOG_FORMATTER",
     "LOG_LEVEL",
+    "LOG_LEVELS",
     "LOG_SHORT_NAMES",
     "LOG_STDOUT",
     "LOG_VERSIONS",
@@ -445,6 +446,7 @@ LOG_FILE_APPEND = True
 LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
 LOG_FORMATTER = "scrapy.logformatter.LogFormatter"
 LOG_LEVEL = "DEBUG"
+LOG_LEVELS: dict[str, str] = {}
 LOG_SHORT_NAMES = False
 LOG_STDOUT = False
 LOG_VERSIONS = [

scrapy/utils/log.py

@@ -62,7 +62,7 @@ class TopLevelFormatter(logging.Filter):
         return True
 
 
-DEFAULT_LOGGING = {
+DEFAULT_LOGGING: dict[str, Any] = {
     "version": 1,
     "disable_existing_loggers": False,
     "loggers": {
@@ -106,7 +106,8 @@ def configure_logging(
     This function does:
 
     - Route warnings and twisted logging through Python standard logging
-    - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
+    - Set log levels for the Scrapy, Twisted and third-party loggers, which
+      :setting:`LOG_LEVELS` can override
     - Route stdout to log if LOG_STDOUT setting is True
 
     When ``install_root_handler`` is True (default), this function also
@@ -122,11 +123,11 @@ def configure_logging(
     observer = twisted_log.PythonLoggingObserver("twisted")
     observer.start()
 
-    dictConfig(DEFAULT_LOGGING)
-
     if isinstance(settings, dict) or settings is None:
         settings = Settings(settings)
 
+    _configure_logger_levels(settings)
+
     if settings.getbool("LOG_STDOUT"):
         sys.stdout = StreamLogger(logging.getLogger("stdout"))
 
@@ -134,6 +135,13 @@ def configure_logging(
         install_scrapy_root_handler(settings)
 
 
+def _configure_logger_levels(settings: Settings) -> None:
+    loggers = DEFAULT_LOGGING["loggers"] | {
+        name: {"level": level} for name, level in settings.getdict("LOG_LEVELS").items()
+    }
+    dictConfig(DEFAULT_LOGGING | {"loggers": loggers})
+
+
 _scrapy_root_handler: logging.Handler | None = None
 
 

tests/test_crawler.py

@@ -528,6 +528,20 @@ class TestCrawlerLogging:
         get_crawler(MySpider)
         assert get_scrapy_root_handler() is None
 
+    def test_spider_custom_settings_log_levels(self) -> None:
+        class MySpider(scrapy.Spider):
+            name = "spider"
+            custom_settings = {"LOG_LEVELS": {"httpx": "DEBUG"}}
+
+        try:
+            configure_logging()
+            assert logging.getLogger("httpx").level == logging.WARNING
+            get_crawler(MySpider)
+            assert logging.getLogger("httpx").level == logging.DEBUG
+        finally:
+            _uninstall_scrapy_root_handler()
+            logging.getLogger("httpx").setLevel(logging.NOTSET)
+
     @coroutine_test
     async def test_spider_custom_settings_log_level(self, tmp_path: Path) -> None:
         log_file = Path(tmp_path, "log.txt")

tests/test_utils_log.py

@@ -15,6 +15,7 @@ from scrapy.utils.log import (
     SpiderLoggerAdapter,
     StreamLogger,
     TopLevelFormatter,
+    configure_logging,
     failure_to_exc_info,
 )
 from scrapy.utils.test import get_crawler
@@ -156,6 +157,29 @@ def test_spider_logger_adapter_process(
     assert result_kwargs == expected_extra
 
 
+class TestLogLevels:
+    @pytest.fixture(autouse=True)
+    def restore_levels(self) -> Generator[None]:
+        yield
+        for name in ("httpx", "boto3"):
+            logging.getLogger(name).setLevel(logging.NOTSET)
+
+    def test_defaults(self) -> None:
+        configure_logging(install_root_handler=False)
+
+        assert logging.getLogger("httpx").level == logging.WARNING
+        assert logging.getLogger("boto3").level == logging.NOTSET
+
+    def test_override(self) -> None:
+        configure_logging(
+            {"LOG_LEVELS": {"httpx": "DEBUG", "boto3": "ERROR"}},
+            install_root_handler=False,
+        )
+
+        assert logging.getLogger("httpx").level == logging.DEBUG
+        assert logging.getLogger("boto3").level == logging.ERROR
+
+
 class TestLogging:
     @pytest.fixture
     def log_stream(self) -> StringIO: