From abbf3b95fcdf814c3926029ea80e4c33bd866c71 Mon Sep 17 00:00:00 2001 From: Lions-1 <128939687+Lions-1@users.noreply.github.com> Date: Fri, 19 Jun 2026 05:41:26 +0100 Subject: [PATCH] tests: integration coverage for memusage extension (#7017) --- scrapy/extensions/memusage.py | 6 +- tests/test_extension_memusage.py | 115 +++++++++++++++++++++++++++++++ tests/utils/__init__.py | 17 +++++ 3 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 tests/test_extension_memusage.py diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 01af02ba3..1444c8941 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -113,7 +113,7 @@ class MemoryUsage: {"memusage": mem}, extra={"crawler": self.crawler}, ) - if self.notify_mails: + if self.notify_mails: # pragma: no cover subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " f"memory usage exceeded {mem}MiB at {socket.gethostname()}" @@ -146,7 +146,7 @@ class MemoryUsage: {"memusage": mem}, extra={"crawler": self.crawler}, ) - if self.notify_mails: + if self.notify_mails: # pragma: no cover subj = ( f"{self.crawler.settings['BOT_NAME']} warning: " f"memory usage reached {mem}MiB at {socket.gethostname()}" @@ -155,7 +155,7 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True - def _send_report(self, rcpts: list[str], subject: str) -> None: + def _send_report(self, rcpts: list[str], subject: str) -> None: # pragma: no cover """send notification mail with some additional useful info""" assert self.crawler.engine assert self.crawler.stats diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py new file mode 100644 index 000000000..a474725d8 --- /dev/null +++ b/tests/test_extension_memusage.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import logging +import sys + +import pytest + +from scrapy import signals +from scrapy.core import engine as engine_mod +from scrapy.exceptions import NotConfigured +from scrapy.extensions import memusage as memusage_mod +from scrapy.extensions.memusage import MemoryUsage +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from tests.utils import OneShotLoop +from tests.utils.decorators import coroutine_test + +# MemoryUsage relies on the stdlib 'resource' module (not available on Windows) +pytestmark = pytest.mark.skipif( + sys.platform.startswith("win"), + reason="MemoryUsage extension not available on Windows", +) + + +MB = 1024 * 1024 + + +class _LoopSpider(Spider): + name = "loop-data-spider" + + def __init__(self, url: str, loops: int = 60, **kw): + super().__init__(**kw) + self.url = url + self.loops = loops + self.start_urls = [url] + + def parse(self, response): + count = response.meta.get("count", 0) + if count + 1 < self.loops: + yield response.follow( + self.url, callback=self.parse, meta={"count": count + 1} + ) + + +def test_memusage_disabled() -> None: + settings = { + "MEMUSAGE_ENABLED": False, + } + with pytest.raises(NotConfigured): + MemoryUsage.from_crawler(get_crawler(settings_dict=settings)) + + +@coroutine_test +async def test_memusage_limit_closes_spider_with_reason_and_error_log( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = { + "MEMUSAGE_LIMIT_MB": 10, + "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01, + "TELNETCONSOLE_ENABLED": False, + "LOG_LEVEL": "INFO", + } + + # Avoid background LoopingCall that can log after the test finishes. + monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + # Avoid engine start/stop races (the extension stops the engine in engine_started). + monkeypatch.setattr(engine_mod, "create_looping_call", OneShotLoop) + monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 250 * MB) + + crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) + + with caplog.at_level(logging.ERROR, logger="scrapy.extensions.memusage"): + await crawler.crawl_async(url="data:,", loops=100) + + assert crawler.stats + assert crawler.stats.get_value("memusage/limit_reached") == 1 + assert crawler.stats.get_value("finish_reason") == "memusage_exceeded" + assert any( + "memory usage exceeded" in r.getMessage().lower() for r in caplog.records + ) + + +@coroutine_test +async def test_memusage_warning_logs_but_allows_normal_finish( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = { + "MEMUSAGE_WARNING_MB": 50, + "MEMUSAGE_LIMIT_MB": 0, # no hard limit + "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01, + "TELNETCONSOLE_ENABLED": False, + "LOG_LEVEL": "INFO", + } + + # Avoid background LoopingCall that can log after the test finishes. + monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB) + + crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) + + warning_signals: list[int] = [] + + def on_warning_reached() -> None: + warning_signals.append(1) + + crawler.signals.connect(on_warning_reached, signal=signals.memusage_warning_reached) + + with caplog.at_level(logging.WARNING, logger="scrapy.extensions.memusage"): + await crawler.crawl_async(url="data:,", loops=60) + + assert warning_signals == [1] + assert crawler.stats + assert crawler.stats.get_value("memusage/warning_reached") == 1 + assert crawler.stats.get_value("finish_reason") == "finished" + assert any("memory usage reached" in r.getMessage().lower() for r in caplog.records) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 5b7848d54..73c0e3776 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,5 +1,6 @@ import asyncio import os +from collections.abc import Callable from pathlib import Path from twisted.internet.defer import Deferred @@ -31,3 +32,19 @@ def get_script_run_env() -> dict[str, str]: env = os.environ.copy() env["PYTHONPATH"] = pythonpath return env + + +class OneShotLoop: + """Test stub for create_looping_call: run once immediately, no background task.""" + + def __init__(self, func: Callable[[], None]): + self.func = func + self.running = False + + def start(self, _interval: float, now: bool = True) -> None: + self.running = True + if now: + self.func() + + def stop(self) -> None: + self.running = False