From 21b9ba717c1687a889879232f226c75fb4dbe0bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 16 Dec 2024 14:46:23 +0100 Subject: [PATCH] Allow customizing logged software versions (#6582) Co-authored-by: Grammy Jiang Co-authored-by: Andrey Rakhmatullin --- docs/topics/settings.rst | 19 ++++++++++ scrapy/commands/version.py | 4 +- scrapy/settings/default_settings.py | 12 ++++++ scrapy/utils/log.py | 14 +++---- scrapy/utils/versions.py | 59 ++++++++++++++++++----------- tests/test_crawler.py | 26 +++++++++++++ 6 files changed, 103 insertions(+), 31 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index cce4a7b3e..76904a26e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1228,6 +1228,25 @@ Default: ``False`` If ``True``, the logs will just contain the root path. If it is set to ``False`` then it displays the component responsible for the log output +.. setting:: LOG_VERSIONS + +LOG_VERSIONS +------------ + +Default: ``["lxml", "libxml2", "cssselect", "parsel", "w3lib", "Twisted", "Python", "pyOpenSSL", "cryptography", "Platform"]`` + +Logs the installed versions of the specified items. + +An item can be any installed Python package. + +The following special items are also supported: + +- ``libxml2`` + +- ``Platform`` (:func:`platform.platform`) + +- ``Python`` + .. setting:: LOGSTATS_INTERVAL LOGSTATS_INTERVAL diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 571f4fda8..713a78ad9 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -2,7 +2,7 @@ import argparse import scrapy from scrapy.commands import ScrapyCommand -from scrapy.utils.versions import scrapy_components_versions +from scrapy.utils.versions import get_versions class Command(ScrapyCommand): @@ -26,7 +26,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if opts.verbose: - versions = scrapy_components_versions() + versions = get_versions() width = max(len(n) for (n, _) in versions) for name, version in versions: print(f"{name:<{width}} : {version}") diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 89ab21fbe..0bbde118e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -219,6 +219,18 @@ LOG_LEVEL = "DEBUG" LOG_FILE = None LOG_FILE_APPEND = True LOG_SHORT_NAMES = False +LOG_VERSIONS = [ + "lxml", + "libxml2", + "cssselect", + "parsel", + "w3lib", + "Twisted", + "Python", + "pyOpenSSL", + "cryptography", + "Platform", +] SCHEDULER_DEBUG = False diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 6165d1f72..d51231b82 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import pprint import sys from collections.abc import MutableMapping from logging.config import dictConfig @@ -12,7 +13,7 @@ from twisted.python.failure import Failure import scrapy from scrapy.settings import Settings, _SettingsKeyT -from scrapy.utils.versions import scrapy_components_versions +from scrapy.utils.versions import get_versions if TYPE_CHECKING: @@ -174,12 +175,11 @@ def log_scrapy_info(settings: Settings) -> None: "Scrapy %(version)s started (bot: %(bot)s)", {"version": scrapy.__version__, "bot": settings["BOT_NAME"]}, ) - versions = [ - f"{name} {version}" - for name, version in scrapy_components_versions() - if name != "Scrapy" - ] - logger.info("Versions: %(versions)s", {"versions": ", ".join(versions)}) + software = settings.getlist("LOG_VERSIONS") + if not software: + return + versions = pprint.pformat(dict(get_versions(software)), sort_dicts=False) + logger.info(f"Versions:\n{versions}") def log_reactor_info() -> None: diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 996a5cdb3..ff1f9b346 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -1,31 +1,46 @@ +from __future__ import annotations + import platform import sys +from importlib.metadata import version +from warnings import warn -import cryptography -import cssselect import lxml.etree -import parsel -import twisted -import w3lib -import scrapy +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.settings.default_settings import LOG_VERSIONS from scrapy.utils.ssl import get_openssl_version +_DEFAULT_SOFTWARE = ["Scrapy"] + LOG_VERSIONS + + +def _version(item): + lowercase_item = item.lower() + if lowercase_item == "libxml2": + return ".".join(map(str, lxml.etree.LIBXML_VERSION)) + if lowercase_item == "platform": + return platform.platform() + if lowercase_item == "pyopenssl": + return get_openssl_version() + if lowercase_item == "python": + return sys.version.replace("\n", "- ") + return version(item) + + +def get_versions( + software: list | None = None, +) -> list[tuple[str, str]]: + software = software or _DEFAULT_SOFTWARE + return [(item, _version(item)) for item in software] + def scrapy_components_versions() -> list[tuple[str, str]]: - lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) - libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) - - return [ - ("Scrapy", scrapy.__version__), - ("lxml", lxml_version), - ("libxml2", libxml2_version), - ("cssselect", cssselect.__version__), - ("parsel", parsel.__version__), - ("w3lib", w3lib.__version__), - ("Twisted", twisted.version.short()), - ("Python", sys.version.replace("\n", "- ")), - ("pyOpenSSL", get_openssl_version()), - ("cryptography", cryptography.__version__), - ("Platform", platform.platform()), - ] + warn( + ( + "scrapy.utils.versions.scrapy_components_versions() is deprecated, " + "use scrapy.utils.versions.get_versions() instead." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + return get_versions() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index a77531f62..f3e5ebf5d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,6 +1,7 @@ import logging import os import platform +import re import signal import subprocess import sys @@ -923,3 +924,28 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): log, ) self.assertIn("DEBUG: Using asyncio event loop", log) + + +@mark.parametrize( + ["settings", "items"], + ( + ({}, default_settings.LOG_VERSIONS), + ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]), + ({"LOG_VERSIONS": []}, None), + ), +) +def test_log_scrapy_info(settings, items, caplog): + with caplog.at_level("INFO"): + CrawlerProcess(settings) + assert ( + caplog.records[0].getMessage() + == f"Scrapy {scrapy.__version__} started (bot: scrapybot)" + ), repr(caplog.records[0].msg) + if not items: + assert len(caplog.records) == 1 + return + version_string = caplog.records[1].getMessage() + expected_items_pattern = "',\n '".join( + f"{item}': '[^']+('\n +'[^']+)*" for item in items + ) + assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string)