diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 65ee77258..351c5890b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -377,12 +377,13 @@ is ``True``. Logging settings ---------------- -**Logging settings** are settings that configure the global root logging -handler installed by :func:`~scrapy.utils.log.configure_logging`. +**Logging settings** are settings that configure logging process-wide, mostly +through the global root logging handler installed by +:func:`~scrapy.utils.log.configure_logging`. -These settings can be defined from a spider. However, because only 1 root -logging handler is active per process, these settings cannot use a different -value per spider when :ref:`running multiple spiders in the same process +These settings can be defined from a spider. However, because logging +configuration is global, these settings cannot use a different value per +spider when :ref:`running multiple spiders in the same process `. These settings are: @@ -394,6 +395,7 @@ These settings are: - :setting:`LOG_FILE_APPEND` - :setting:`LOG_FORMAT` - :setting:`LOG_LEVEL` +- :setting:`LOG_LEVELS` - :setting:`LOG_SHORT_NAMES` - :setting:`LOG_STDOUT` @@ -1615,6 +1617,30 @@ INFO, DEBUG. For more info see :ref:`topics-logging`. .. note:: This is a :ref:`logging setting `. +.. setting:: LOG_LEVELS + +LOG_LEVELS +---------- + +Default: ``{}`` + +.. versionadded:: VERSION + +Minimum level to log for specific loggers. + +It takes precedence over the levels that Scrapy sets by default:: + + { + "filelock": "ERROR", + "hpack": "ERROR", + "httpcore": "ERROR", + "httpx": "WARNING", + "scrapy": "DEBUG", + "twisted": "ERROR", + } + +.. note:: This is a :ref:`logging setting `. + .. setting:: LOG_STDOUT LOG_STDOUT diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e2f726519..58b24defb 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -22,6 +22,7 @@ from scrapy.signalmanager import SignalManager from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader from scrapy.utils.defer import deferred_from_coro from scrapy.utils.log import ( + _configure_logger_levels, configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, @@ -74,7 +75,7 @@ class Crawler: self.spidercls: type[Spider] = spidercls self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self._update_root_log_handler() + self._update_logging() self.addons: AddonManager = AddonManager(self) self.signals: SignalManager = SignalManager(self) @@ -90,9 +91,10 @@ class Crawler: self.spider: Spider | None = None self.engine: ExecutionEngine | None = None - def _update_root_log_handler(self) -> None: + def _update_logging(self) -> None: if get_scrapy_root_handler() is not None: # scrapy root handler already installed: update it with new settings + _configure_logger_levels(self.settings) install_scrapy_root_handler(self.settings) def _apply_settings(self) -> None: @@ -219,7 +221,7 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) self._apply_settings() - self._update_root_log_handler() + self._update_logging() self.engine = self._create_engine() yield deferred_from_coro(self.engine.open_spider_async()) yield deferred_from_coro(self.engine.start_async()) @@ -249,7 +251,7 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) self._apply_settings() - self._update_root_log_handler() + self._update_logging() self.engine = self._create_engine() await self.engine.open_spider_async() await self.engine.start_async() diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a44b36c8a..fc20d4f90 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -144,6 +144,7 @@ __all__ = [ "LOG_FORMAT", "LOG_FORMATTER", "LOG_LEVEL", + "LOG_LEVELS", "LOG_SHORT_NAMES", "LOG_STDOUT", "LOG_VERSIONS", @@ -445,6 +446,7 @@ LOG_FILE_APPEND = True LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s" LOG_FORMATTER = "scrapy.logformatter.LogFormatter" LOG_LEVEL = "DEBUG" +LOG_LEVELS: dict[str, str] = {} LOG_SHORT_NAMES = False LOG_STDOUT = False LOG_VERSIONS = [ diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 7645b235e..59ab5d521 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -62,7 +62,7 @@ class TopLevelFormatter(logging.Filter): return True -DEFAULT_LOGGING = { +DEFAULT_LOGGING: dict[str, Any] = { "version": 1, "disable_existing_loggers": False, "loggers": { @@ -106,7 +106,8 @@ def configure_logging( This function does: - Route warnings and twisted logging through Python standard logging - - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively + - Set log levels for the Scrapy, Twisted and third-party loggers, which + :setting:`LOG_LEVELS` can override - Route stdout to log if LOG_STDOUT setting is True When ``install_root_handler`` is True (default), this function also @@ -122,11 +123,11 @@ def configure_logging( observer = twisted_log.PythonLoggingObserver("twisted") observer.start() - dictConfig(DEFAULT_LOGGING) - if isinstance(settings, dict) or settings is None: settings = Settings(settings) + _configure_logger_levels(settings) + if settings.getbool("LOG_STDOUT"): sys.stdout = StreamLogger(logging.getLogger("stdout")) @@ -134,6 +135,13 @@ def configure_logging( install_scrapy_root_handler(settings) +def _configure_logger_levels(settings: Settings) -> None: + loggers = DEFAULT_LOGGING["loggers"] | { + name: {"level": level} for name, level in settings.getdict("LOG_LEVELS").items() + } + dictConfig(DEFAULT_LOGGING | {"loggers": loggers}) + + _scrapy_root_handler: logging.Handler | None = None diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 358f20ed7..ceb96da20 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -528,6 +528,20 @@ class TestCrawlerLogging: get_crawler(MySpider) assert get_scrapy_root_handler() is None + def test_spider_custom_settings_log_levels(self) -> None: + class MySpider(scrapy.Spider): + name = "spider" + custom_settings = {"LOG_LEVELS": {"httpx": "DEBUG"}} + + try: + configure_logging() + assert logging.getLogger("httpx").level == logging.WARNING + get_crawler(MySpider) + assert logging.getLogger("httpx").level == logging.DEBUG + finally: + _uninstall_scrapy_root_handler() + logging.getLogger("httpx").setLevel(logging.NOTSET) + @coroutine_test async def test_spider_custom_settings_log_level(self, tmp_path: Path) -> None: log_file = Path(tmp_path, "log.txt") diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 7f5301387..d82d133ef 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -15,6 +15,7 @@ from scrapy.utils.log import ( SpiderLoggerAdapter, StreamLogger, TopLevelFormatter, + configure_logging, failure_to_exc_info, ) from scrapy.utils.test import get_crawler @@ -156,6 +157,29 @@ def test_spider_logger_adapter_process( assert result_kwargs == expected_extra +class TestLogLevels: + @pytest.fixture(autouse=True) + def restore_levels(self) -> Generator[None]: + yield + for name in ("httpx", "boto3"): + logging.getLogger(name).setLevel(logging.NOTSET) + + def test_defaults(self) -> None: + configure_logging(install_root_handler=False) + + assert logging.getLogger("httpx").level == logging.WARNING + assert logging.getLogger("boto3").level == logging.NOTSET + + def test_override(self) -> None: + configure_logging( + {"LOG_LEVELS": {"httpx": "DEBUG", "boto3": "ERROR"}}, + install_root_handler=False, + ) + + assert logging.getLogger("httpx").level == logging.DEBUG + assert logging.getLogger("boto3").level == logging.ERROR + + class TestLogging: @pytest.fixture def log_stream(self) -> StringIO: