make --profile actually useful

docs/topics/commands.rst

@@ -209,6 +209,48 @@ Project-only commands:
 * :command:`parse`
 * :command:`bench`
 
+.. _topics-global-options:
+
+Global options
+==============
+
+These options are available for all Scrapy commands:
+
+``--logfile FILE``
+    Log output to *FILE* instead of stderr.
+
+``-L LEVEL``, ``--loglevel LEVEL``
+    Set the logging level. Available levels (in increasing severity):
+    ``DEBUG``, ``INFO``, ``WARNING``, ``ERROR``, ``CRITICAL``.
+    Default: ``WARNING``.
+
+``--nolog``
+    Disable logging completely.
+
+``--pidfile FILE``
+    Write the process ID to *FILE*.
+
+``-s NAME=VALUE``, ``--set NAME=VALUE``
+    Override a Scrapy setting (may be repeated)::
+
+        scrapy crawl myspider -s LOG_LEVEL=DEBUG -s CONCURRENT_REQUESTS=4
+
+``--profile FILE``
+    Run the command under ``cProfile`` and write the binary stats dump to
+    *FILE*. After the run, a human-readable summary of the top 20 functions
+    sorted by cumulative time is printed to the log at ``INFO`` level.
+
+    Usage example::
+
+        scrapy crawl myspider --profile stats.prof
+
+    To inspect the saved binary dump later::
+
+        python -c "import pstats; pstats.Stats('stats.prof').sort_stats('cumulative').print_stats(20)"
+
+``--pdb``
+    Enable ``pdb`` on failure.
+
 .. command:: startproject
 
 startproject

scrapy/cmdline.py

@@ -3,7 +3,10 @@ from __future__ import annotations
 import argparse
 import cProfile
 import inspect
+import io
+import logging
 import os
+import pstats
 import sys
 from importlib.metadata import entry_points
 from typing import TYPE_CHECKING, ParamSpec
@@ -24,6 +27,8 @@ if TYPE_CHECKING:
 
 _P = ParamSpec("_P")
 
+logger = logging.getLogger(__name__)
+
 
 class ScrapyArgumentParser(argparse.ArgumentParser):
     def _parse_optional(
@@ -232,6 +237,20 @@ def _run_command_profiled(
     p.runctx("cmd.run(args, opts)", globals(), loc)
     if opts.profile:
         p.dump_stats(opts.profile)
+    _log_profile_stats(p)
+
+
+def _log_profile_stats(
+    p: cProfile.Profile,
+    sort: str = "cumulative",
+    lines: int = 20,
+) -> None:
+    """Log a human-readable cProfile stats summary at WARNING level."""
+    stream = io.StringIO()
+    ps = pstats.Stats(p, stream=stream)
+    ps.sort_stats(sort)
+    ps.print_stats(lines)
+    logger.warning("cProfile stats:\n%s", stream.getvalue())
 
 
 if __name__ == "__main__":

scrapy/commands/__init__.py

@@ -104,12 +104,6 @@ class ScrapyCommand(ABC):
         group.add_argument(
             "--nolog", action="store_true", help="disable logging completely"
         )
-        group.add_argument(
-            "--profile",
-            metavar="FILE",
-            default=None,
-            help="write python cProfile stats to FILE",
-        )
         group.add_argument("--pidfile", metavar="FILE", help="write process ID to FILE")
         group.add_argument(
             "-s",
@@ -119,6 +113,12 @@ class ScrapyCommand(ABC):
             metavar="NAME=VALUE",
             help="set/override setting (may be repeated)",
         )
+        group.add_argument(
+            "--profile",
+            metavar="FILE",
+            default=None,
+            help="write python cProfile stats to FILE and print a stats summary",
+        )
         group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
 
     def process_options(self, args: list[str], opts: argparse.Namespace) -> None:

tests/test_cmdline/__init__.py

@@ -22,38 +22,44 @@ class TestCmdline:
         encoding = sys.stdout.encoding or "utf-8"
         args = (sys.executable, "-m", "scrapy.cmdline", *new_args)
         proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)
-        comm = proc.communicate()[0].strip()
-        return comm.decode(encoding)
+        stdout, stderr = proc.communicate()
+        return stdout.strip().decode(encoding), stderr.strip().decode(encoding)
 
     def test_default_settings(self):
-        assert self._execute("settings", "--get", "TEST1") == "default"
+        stdout, _ = self._execute("settings", "--get", "TEST1")
+        assert stdout == "default"
 
     def test_override_settings_using_set_arg(self):
-        assert (
-            self._execute("settings", "--get", "TEST1", "-s", "TEST1=override")
-            == "override"
+        stdout, _ = self._execute(
+            "settings", "--get", "TEST1", "-s", "TEST1=override"
         )
+        assert stdout == "override"
 
     def test_profiling(self):
         path = Path(tempfile.mkdtemp())
         filename = path / "res.prof"
         try:
-            self._execute("version", "--profile", str(filename))
+            _, log_output = self._execute("version", "--profile", str(filename))
+            # Binary dump file must be created
             assert filename.exists()
+            # Binary dump must contain valid cProfile data
             out = StringIO()
             stats = pstats.Stats(str(filename), stream=out)
             stats.print_stats()
             out.seek(0)
-            stats = out.read()
-            assert str(Path("scrapy", "commands", "version.py")) in stats
-            assert "tottime" in stats
+            stats_text = out.read()
+            assert str(Path("scrapy", "commands", "version.py")) in stats_text
+            assert "tottime" in stats_text
+            # Human-readable summary must appear in the log (stderr)
+            assert "cProfile stats" in log_output
+            assert "cumtime" in log_output
         finally:
             shutil.rmtree(path)
 
     def test_override_dict_settings(self):
         EXT_PATH = "tests.test_cmdline.extensions.DummyExtension"
         EXTENSIONS = {EXT_PATH: 200}
-        settingsstr = self._execute(
+        stdout, _ = self._execute(
             "settings",
             "--get",
             "EXTENSIONS",
@@ -61,14 +67,16 @@ class TestCmdline:
             "EXTENSIONS=" + json.dumps(EXTENSIONS),
         )
         # XXX: There's gotta be a smarter way to do this...
-        assert "..." not in settingsstr
+        assert "..." not in stdout
         for char in ("'", "<", ">"):
-            settingsstr = settingsstr.replace(char, '"')
-        settingsdict = json.loads(settingsstr)
+            stdout = stdout.replace(char, '"')
+        settingsdict = json.loads(stdout)
         assert set(settingsdict.keys()) == set(EXTENSIONS.keys())
         assert settingsdict[EXT_PATH] == 200
 
     def test_pathlib_path_as_feeds_key(self):
-        assert self._execute("settings", "--get", "FEEDS") == json.dumps(
+        stdout, _ = self._execute("settings", "--get", "FEEDS")
+        assert stdout == json.dumps(
             {"items.csv": {"format": "csv", "fields": ["price", "name"]}}
         )
+