From 7a80c63d8fd6c4d82098edb98a0472b74b7f3687 Mon Sep 17 00:00:00 2001 From: Abhijeet Sharma Date: Fri, 3 Jul 2026 15:54:20 +0530 Subject: [PATCH] make --profile actually useful --- docs/topics/commands.rst | 42 ++++++++++++++++++++++++++++++++++ scrapy/cmdline.py | 19 +++++++++++++++ scrapy/commands/__init__.py | 12 +++++----- tests/test_cmdline/__init__.py | 38 ++++++++++++++++++------------ 4 files changed, 90 insertions(+), 21 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8d1351eb9..0dc920f3e 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -209,6 +209,48 @@ Project-only commands: * :command:`parse` * :command:`bench` +.. _topics-global-options: + +Global options +============== + +These options are available for all Scrapy commands: + +``--logfile FILE`` + Log output to *FILE* instead of stderr. + +``-L LEVEL``, ``--loglevel LEVEL`` + Set the logging level. Available levels (in increasing severity): + ``DEBUG``, ``INFO``, ``WARNING``, ``ERROR``, ``CRITICAL``. + Default: ``WARNING``. + +``--nolog`` + Disable logging completely. + +``--pidfile FILE`` + Write the process ID to *FILE*. + +``-s NAME=VALUE``, ``--set NAME=VALUE`` + Override a Scrapy setting (may be repeated):: + + scrapy crawl myspider -s LOG_LEVEL=DEBUG -s CONCURRENT_REQUESTS=4 + +``--profile FILE`` + Run the command under ``cProfile`` and write the binary stats dump to + *FILE*. After the run, a human-readable summary of the top 20 functions + sorted by cumulative time is printed to the log at ``INFO`` level. + + Usage example:: + + scrapy crawl myspider --profile stats.prof + + To inspect the saved binary dump later:: + + python -c "import pstats; pstats.Stats('stats.prof').sort_stats('cumulative').print_stats(20)" + +``--pdb`` + Enable ``pdb`` on failure. + .. command:: startproject startproject diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 6c306afdb..39acd6972 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -3,7 +3,10 @@ from __future__ import annotations import argparse import cProfile import inspect +import io +import logging import os +import pstats import sys from importlib.metadata import entry_points from typing import TYPE_CHECKING, ParamSpec @@ -24,6 +27,8 @@ if TYPE_CHECKING: _P = ParamSpec("_P") +logger = logging.getLogger(__name__) + class ScrapyArgumentParser(argparse.ArgumentParser): def _parse_optional( @@ -232,6 +237,20 @@ def _run_command_profiled( p.runctx("cmd.run(args, opts)", globals(), loc) if opts.profile: p.dump_stats(opts.profile) + _log_profile_stats(p) + + +def _log_profile_stats( + p: cProfile.Profile, + sort: str = "cumulative", + lines: int = 20, +) -> None: + """Log a human-readable cProfile stats summary at WARNING level.""" + stream = io.StringIO() + ps = pstats.Stats(p, stream=stream) + ps.sort_stats(sort) + ps.print_stats(lines) + logger.warning("cProfile stats:\n%s", stream.getvalue()) if __name__ == "__main__": diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index d9c919db9..ef9de0ca1 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -104,12 +104,6 @@ class ScrapyCommand(ABC): group.add_argument( "--nolog", action="store_true", help="disable logging completely" ) - group.add_argument( - "--profile", - metavar="FILE", - default=None, - help="write python cProfile stats to FILE", - ) group.add_argument("--pidfile", metavar="FILE", help="write process ID to FILE") group.add_argument( "-s", @@ -119,6 +113,12 @@ class ScrapyCommand(ABC): metavar="NAME=VALUE", help="set/override setting (may be repeated)", ) + group.add_argument( + "--profile", + metavar="FILE", + default=None, + help="write python cProfile stats to FILE and print a stats summary", + ) group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args: list[str], opts: argparse.Namespace) -> None: diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 98a85bc17..d5f7c8fc6 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -22,38 +22,44 @@ class TestCmdline: encoding = sys.stdout.encoding or "utf-8" args = (sys.executable, "-m", "scrapy.cmdline", *new_args) proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) - comm = proc.communicate()[0].strip() - return comm.decode(encoding) + stdout, stderr = proc.communicate() + return stdout.strip().decode(encoding), stderr.strip().decode(encoding) def test_default_settings(self): - assert self._execute("settings", "--get", "TEST1") == "default" + stdout, _ = self._execute("settings", "--get", "TEST1") + assert stdout == "default" def test_override_settings_using_set_arg(self): - assert ( - self._execute("settings", "--get", "TEST1", "-s", "TEST1=override") - == "override" + stdout, _ = self._execute( + "settings", "--get", "TEST1", "-s", "TEST1=override" ) + assert stdout == "override" def test_profiling(self): path = Path(tempfile.mkdtemp()) filename = path / "res.prof" try: - self._execute("version", "--profile", str(filename)) + _, log_output = self._execute("version", "--profile", str(filename)) + # Binary dump file must be created assert filename.exists() + # Binary dump must contain valid cProfile data out = StringIO() stats = pstats.Stats(str(filename), stream=out) stats.print_stats() out.seek(0) - stats = out.read() - assert str(Path("scrapy", "commands", "version.py")) in stats - assert "tottime" in stats + stats_text = out.read() + assert str(Path("scrapy", "commands", "version.py")) in stats_text + assert "tottime" in stats_text + # Human-readable summary must appear in the log (stderr) + assert "cProfile stats" in log_output + assert "cumtime" in log_output finally: shutil.rmtree(path) def test_override_dict_settings(self): EXT_PATH = "tests.test_cmdline.extensions.DummyExtension" EXTENSIONS = {EXT_PATH: 200} - settingsstr = self._execute( + stdout, _ = self._execute( "settings", "--get", "EXTENSIONS", @@ -61,14 +67,16 @@ class TestCmdline: "EXTENSIONS=" + json.dumps(EXTENSIONS), ) # XXX: There's gotta be a smarter way to do this... - assert "..." not in settingsstr + assert "..." not in stdout for char in ("'", "<", ">"): - settingsstr = settingsstr.replace(char, '"') - settingsdict = json.loads(settingsstr) + stdout = stdout.replace(char, '"') + settingsdict = json.loads(stdout) assert set(settingsdict.keys()) == set(EXTENSIONS.keys()) assert settingsdict[EXT_PATH] == 200 def test_pathlib_path_as_feeds_key(self): - assert self._execute("settings", "--get", "FEEDS") == json.dumps( + stdout, _ = self._execute("settings", "--get", "FEEDS") + assert stdout == json.dumps( {"items.csv": {"format": "csv", "fields": ["price", "name"]}} ) +