From 28c3b6fff34d65e0cfe1cf2688fb39b747ec6d20 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 17 Jul 2026 19:54:41 +0500 Subject: [PATCH 1/3] Tests for extensions.debug and extensions.memdebug. --- tests/test_extension_debug.py | 132 +++++++++++++++++++++++++++++++ tests/test_extension_memdebug.py | 47 +++++++++++ 2 files changed, 179 insertions(+) create mode 100644 tests/test_extension_debug.py create mode 100644 tests/test_extension_memdebug.py diff --git a/tests/test_extension_debug.py b/tests/test_extension_debug.py new file mode 100644 index 000000000..42d78688f --- /dev/null +++ b/tests/test_extension_debug.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import logging +import os +import signal +import sys +import threading +from typing import TYPE_CHECKING +from unittest import mock + +import pytest + +from scrapy.extensions.debug import Debugger, StackTraceDump +from scrapy.spiders import Spider +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from collections.abc import Generator + + +@pytest.fixture(autouse=True) +def preserve_signal_handlers() -> Generator[None]: + """Restore the signal handlers that the extensions replace.""" + signums = [ + getattr(signal, name) + for name in ("SIGUSR2", "SIGQUIT") + if hasattr(signal, name) + ] + handlers = {signum: signal.getsignal(signum) for signum in signums} + yield + for signum, handler in handlers.items(): + signal.signal(signum, handler) + + +class SignalSpider(Spider): + name = "signal_spider" + start_urls = ["data:,"] + + def parse(self, response): + os.kill(os.getpid(), signal.SIGUSR2) + return [] + + +@pytest.mark.skipif( + sys.platform == "win32", reason="SIGUSR2 and SIGQUIT are POSIX-only" +) +def test_stacktracedump_installs_signal_handlers() -> None: + crawler = get_crawler() + ext = StackTraceDump.from_crawler(crawler) + assert signal.getsignal(signal.SIGUSR2) == ext.dump_stacktrace # pylint: disable=comparison-with-callable + assert signal.getsignal(signal.SIGQUIT) == ext.dump_stacktrace # pylint: disable=comparison-with-callable + + +def test_stacktracedump_works_without_signal_support( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # simulate win32 platforms, which don't support SIGUSR signals + monkeypatch.delattr(signal, "SIGUSR2", raising=False) + ext = StackTraceDump.from_crawler(get_crawler()) + assert isinstance(ext, StackTraceDump) + + +def test_stacktracedump_dump_stacktrace(caplog: pytest.LogCaptureFixture) -> None: + crawler = get_crawler() + crawler.engine = mock.Mock() + ext = StackTraceDump.from_crawler(crawler) + spider = DefaultSpider() + with caplog.at_level(logging.INFO, logger="scrapy.extensions.debug"): + ext.dump_stacktrace(0, None) + assert len(caplog.records) == 1 + message = caplog.records[0].getMessage() + assert "Dumping stack trace and engine status" in message + assert "Execution engine status" in message + assert "Live References" in message + assert type(spider).__name__ in message + assert "# Thread: MainThread" in message + assert getattr(caplog.records[0], "crawler", None) is crawler + + +def test_stacktracedump_thread_stacks() -> None: + ext = StackTraceDump.from_crawler(get_crawler()) + stop = threading.Event() + thread = threading.Thread(target=stop.wait, name="dump-test-thread") + thread.start() + try: + stacks = ext._thread_stacks() + finally: + stop.set() + thread.join() + assert "# Thread: MainThread" in stacks + assert "# Thread: dump-test-thread" in stacks + + +@pytest.mark.skipif(sys.platform == "win32", reason="SIGUSR2 is POSIX-only") +@coroutine_test +async def test_stacktracedump_dumps_on_signal(caplog: pytest.LogCaptureFixture) -> None: + settings = { + "EXTENSIONS": {"scrapy.extensions.debug.StackTraceDump": 0}, + "LOG_LEVEL": "INFO", + } + crawler = get_crawler(spidercls=SignalSpider, settings_dict=settings) + with caplog.at_level(logging.INFO, logger="scrapy.extensions.debug"): + await crawler.crawl_async() + assert len(caplog.records) == 1 + message = caplog.records[0].getMessage() + assert "Dumping stack trace and engine status" in message + assert "engine.spider.name" in message + assert "signal_spider" in message + + +@pytest.mark.skipif(sys.platform == "win32", reason="SIGUSR2 is POSIX-only") +def test_debugger_installs_signal_handler() -> None: + ext = Debugger() + assert signal.getsignal(signal.SIGUSR2) == ext._enter_debugger # pylint: disable=comparison-with-callable + + +def test_debugger_works_without_signal_support(monkeypatch: pytest.MonkeyPatch) -> None: + # simulate win32 platforms, which don't support SIGUSR signals + monkeypatch.delattr(signal, "SIGUSR2", raising=False) + ext = Debugger() + assert isinstance(ext, Debugger) + + +def test_debugger_enter_debugger(monkeypatch: pytest.MonkeyPatch) -> None: + pdb_cls = mock.Mock() + monkeypatch.setattr("scrapy.extensions.debug.Pdb", pdb_cls) + ext = Debugger() + frame = sys._getframe() + ext._enter_debugger(0, frame) + pdb_cls.return_value.set_trace.assert_called_once_with(frame.f_back) diff --git a/tests/test_extension_memdebug.py b/tests/test_extension_memdebug.py new file mode 100644 index 000000000..50ca51db2 --- /dev/null +++ b/tests/test_extension_memdebug.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import gc + +import pytest + +from scrapy.exceptions import NotConfigured +from scrapy.extensions.memdebug import MemoryDebugger +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler +from scrapy.utils.trackref import object_ref +from tests.utils.decorators import coroutine_test + + +def test_disabled_by_default() -> None: + with pytest.raises(NotConfigured): + MemoryDebugger.from_crawler(get_crawler()) + + +def test_spider_closed_sets_stats() -> None: + crawler = get_crawler(settings_dict={"MEMDEBUG_ENABLED": True}) + ext = MemoryDebugger.from_crawler(crawler) + + class TrackedObject(object_ref): + pass + + class CollectedObject(object_ref): + pass + + tracked = [TrackedObject(), TrackedObject()] + CollectedObject() + + ext.spider_closed(DefaultSpider(), "finished") + + assert crawler.stats + assert crawler.stats.get_value("memdebug/gc_garbage_count") == len(gc.garbage) + assert crawler.stats.get_value("memdebug/live_refs/TrackedObject") == len(tracked) + assert crawler.stats.get_value("memdebug/live_refs/CollectedObject") is None + + +@coroutine_test +async def test_crawl_sets_stats() -> None: + crawler = get_crawler(DefaultSpider, settings_dict={"MEMDEBUG_ENABLED": True}) + await crawler.crawl_async() + assert crawler.stats + assert crawler.stats.get_value("memdebug/gc_garbage_count") is not None + assert crawler.stats.get_value("memdebug/live_refs/DefaultSpider") == 1 From e3d9672f6b20473651da57bb68bd6c710143c05c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 17 Jul 2026 20:21:48 +0500 Subject: [PATCH 2/3] Use a unique class in test_crawl_sets_stats. --- tests/test_extension_memdebug.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_extension_memdebug.py b/tests/test_extension_memdebug.py index 50ca51db2..e32989aec 100644 --- a/tests/test_extension_memdebug.py +++ b/tests/test_extension_memdebug.py @@ -40,8 +40,12 @@ def test_spider_closed_sets_stats() -> None: @coroutine_test async def test_crawl_sets_stats() -> None: - crawler = get_crawler(DefaultSpider, settings_dict={"MEMDEBUG_ENABLED": True}) + # unique class so that other tests don't pollute live_refs + class MemDebugSpider(DefaultSpider): + pass + + crawler = get_crawler(MemDebugSpider, settings_dict={"MEMDEBUG_ENABLED": True}) await crawler.crawl_async() assert crawler.stats assert crawler.stats.get_value("memdebug/gc_garbage_count") is not None - assert crawler.stats.get_value("memdebug/live_refs/DefaultSpider") == 1 + assert crawler.stats.get_value("memdebug/live_refs/MemDebugSpider") == 1 From 38610a9e8ef9f938c4a0d52da70da0e0411eb87f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 17 Jul 2026 20:39:05 +0500 Subject: [PATCH 3/3] Don't assume a single log message. --- tests/test_extension_debug.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_extension_debug.py b/tests/test_extension_debug.py index 42d78688f..a54807c1c 100644 --- a/tests/test_extension_debug.py +++ b/tests/test_extension_debug.py @@ -103,11 +103,14 @@ async def test_stacktracedump_dumps_on_signal(caplog: pytest.LogCaptureFixture) crawler = get_crawler(spidercls=SignalSpider, settings_dict=settings) with caplog.at_level(logging.INFO, logger="scrapy.extensions.debug"): await crawler.crawl_async() - assert len(caplog.records) == 1 - message = caplog.records[0].getMessage() - assert "Dumping stack trace and engine status" in message - assert "engine.spider.name" in message - assert "signal_spider" in message + for r in caplog.records: + message = r.getMessage() + if "Dumping stack trace and engine status" in message: + assert "Dumping stack trace and engine status" in message + assert "engine.spider.name" in message + assert "signal_spider" in message + return + raise AssertionError("No stack trace dump log message found") @pytest.mark.skipif(sys.platform == "win32", reason="SIGUSR2 is POSIX-only")