Allow customizing logged software versions (#6582)

Co-authored-by: Grammy Jiang <grammy.jiang@gmail.com>
Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>

docs/topics/settings.rst

@@ -1228,6 +1228,25 @@ Default: ``False``
 If ``True``, the logs will just contain the root path. If it is set to ``False``
 then it displays the component responsible for the log output
 
+.. setting:: LOG_VERSIONS
+
+LOG_VERSIONS
+------------
+
+Default: ``["lxml", "libxml2", "cssselect", "parsel", "w3lib", "Twisted", "Python", "pyOpenSSL", "cryptography", "Platform"]``
+
+Logs the installed versions of the specified items.
+
+An item can be any installed Python package.
+
+The following special items are also supported:
+
+-   ``libxml2``
+
+-   ``Platform`` (:func:`platform.platform`)
+
+-   ``Python``
+
 .. setting:: LOGSTATS_INTERVAL
 
 LOGSTATS_INTERVAL

scrapy/commands/version.py

@@ -2,7 +2,7 @@ import argparse
 
 import scrapy
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.versions import scrapy_components_versions
+from scrapy.utils.versions import get_versions
 
 
 class Command(ScrapyCommand):
@@ -26,7 +26,7 @@ class Command(ScrapyCommand):
 
     def run(self, args: list[str], opts: argparse.Namespace) -> None:
         if opts.verbose:
-            versions = scrapy_components_versions()
+            versions = get_versions()
             width = max(len(n) for (n, _) in versions)
             for name, version in versions:
                 print(f"{name:<{width}} : {version}")

scrapy/settings/default_settings.py

@@ -219,6 +219,18 @@ LOG_LEVEL = "DEBUG"
 LOG_FILE = None
 LOG_FILE_APPEND = True
 LOG_SHORT_NAMES = False
+LOG_VERSIONS = [
+    "lxml",
+    "libxml2",
+    "cssselect",
+    "parsel",
+    "w3lib",
+    "Twisted",
+    "Python",
+    "pyOpenSSL",
+    "cryptography",
+    "Platform",
+]
 
 SCHEDULER_DEBUG = False
 

scrapy/utils/log.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import pprint
 import sys
 from collections.abc import MutableMapping
 from logging.config import dictConfig
@@ -12,7 +13,7 @@ from twisted.python.failure import Failure
 
 import scrapy
 from scrapy.settings import Settings, _SettingsKeyT
-from scrapy.utils.versions import scrapy_components_versions
+from scrapy.utils.versions import get_versions
 
 if TYPE_CHECKING:
 
@@ -174,12 +175,11 @@ def log_scrapy_info(settings: Settings) -> None:
         "Scrapy %(version)s started (bot: %(bot)s)",
         {"version": scrapy.__version__, "bot": settings["BOT_NAME"]},
     )
-    versions = [
-        f"{name} {version}"
-        for name, version in scrapy_components_versions()
-        if name != "Scrapy"
-    ]
-    logger.info("Versions: %(versions)s", {"versions": ", ".join(versions)})
+    software = settings.getlist("LOG_VERSIONS")
+    if not software:
+        return
+    versions = pprint.pformat(dict(get_versions(software)), sort_dicts=False)
+    logger.info(f"Versions:\n{versions}")
 
 
 def log_reactor_info() -> None:

scrapy/utils/versions.py

@@ -1,31 +1,46 @@
+from __future__ import annotations
+
 import platform
 import sys
+from importlib.metadata import version
+from warnings import warn
 
-import cryptography
-import cssselect
 import lxml.etree
-import parsel
-import twisted
-import w3lib
 
-import scrapy
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.settings.default_settings import LOG_VERSIONS
 from scrapy.utils.ssl import get_openssl_version
 
+_DEFAULT_SOFTWARE = ["Scrapy"] + LOG_VERSIONS
+
+
+def _version(item):
+    lowercase_item = item.lower()
+    if lowercase_item == "libxml2":
+        return ".".join(map(str, lxml.etree.LIBXML_VERSION))
+    if lowercase_item == "platform":
+        return platform.platform()
+    if lowercase_item == "pyopenssl":
+        return get_openssl_version()
+    if lowercase_item == "python":
+        return sys.version.replace("\n", "- ")
+    return version(item)
+
+
+def get_versions(
+    software: list | None = None,
+) -> list[tuple[str, str]]:
+    software = software or _DEFAULT_SOFTWARE
+    return [(item, _version(item)) for item in software]
+
 
 def scrapy_components_versions() -> list[tuple[str, str]]:
-    lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
-    libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
-
-    return [
-        ("Scrapy", scrapy.__version__),
-        ("lxml", lxml_version),
-        ("libxml2", libxml2_version),
-        ("cssselect", cssselect.__version__),
-        ("parsel", parsel.__version__),
-        ("w3lib", w3lib.__version__),
-        ("Twisted", twisted.version.short()),
-        ("Python", sys.version.replace("\n", "- ")),
-        ("pyOpenSSL", get_openssl_version()),
-        ("cryptography", cryptography.__version__),
-        ("Platform", platform.platform()),
-    ]
+    warn(
+        (
+            "scrapy.utils.versions.scrapy_components_versions() is deprecated, "
+            "use scrapy.utils.versions.get_versions() instead."
+        ),
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+    return get_versions()

tests/test_crawler.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import platform
+import re
 import signal
 import subprocess
 import sys
@@ -923,3 +924,28 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
             log,
         )
         self.assertIn("DEBUG: Using asyncio event loop", log)
+
+
+@mark.parametrize(
+    ["settings", "items"],
+    (
+        ({}, default_settings.LOG_VERSIONS),
+        ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]),
+        ({"LOG_VERSIONS": []}, None),
+    ),
+)
+def test_log_scrapy_info(settings, items, caplog):
+    with caplog.at_level("INFO"):
+        CrawlerProcess(settings)
+    assert (
+        caplog.records[0].getMessage()
+        == f"Scrapy {scrapy.__version__} started (bot: scrapybot)"
+    ), repr(caplog.records[0].msg)
+    if not items:
+        assert len(caplog.records) == 1
+        return
+    version_string = caplog.records[1].getMessage()
+    expected_items_pattern = "',\n '".join(
+        f"{item}': '[^']+('\n +'[^']+)*" for item in items
+    )
+    assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string)