RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

tests: memusage integration, used plain dict with get_crawler() to respect installed reactor

This commit is contained in:
Ayman 2025-08-23 18:32:23 +01:00
parent 95c300ca13
commit 62a133df9a
1 changed files with 17 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
tests/test_memusage_integration.py
View File

@ -8,7 +8,6 @@ from twisted.internet.defer import inlineCallbacks
from scrapy import signals
from scrapy.extensions.memusage import MemoryUsage
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
@ -18,17 +17,6 @@ pytestmark = pytest.mark.skipif(
reason="MemoryUsage extension not available on Windows",
)
MiB = 1024 * 1024
def _installed_reactor_path() -> str:
"""Return the fully qualified path of the *already installed* Twisted reactor.
Local import avoids module-level reactor import (keeps Ruff happy)."""
from twisted.internet import reactor as _reactor # local on purpose
cls = _reactor.__class__
return f"{cls.__module__}.{cls.__name__}"
class _LoopSpider(Spider):
name = "loop-data-spider"
@ -50,21 +38,18 @@ class _LoopSpider(Spider):
@inlineCallbacks
def test_memusage_limit_closes_spider_with_reason_and_error_log(caplog, monkeypatch):
url = "data:,"
settings = Settings(
{
"MEMUSAGE_ENABLED": True,
"MEMUSAGE_LIMIT_MB": 10,
"MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01,
"LOG_LEVEL": "INFO",
}
)
# Respect the reactor already installed by the test env/CI.
settings.set("TWISTED_REACTOR", _installed_reactor_path(), priority="cmdline")
settings = {
"MEMUSAGE_ENABLED": True,
"MEMUSAGE_LIMIT_MB": 10,
"MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01,
"LOG_LEVEL": "INFO",
}
MB = 1024 * 1024
state = {"high": False}
def fake_vsz(self):
return 250 * MiB if state["high"] else 5 * MiB
return 250 * MB if state["high"] else 5 * MB
monkeypatch.setattr(MemoryUsage, "get_virtual_size", fake_vsz)
@ -87,19 +72,16 @@ def test_memusage_limit_closes_spider_with_reason_and_error_log(caplog, monkeypa
@inlineCallbacks
def test_memusage_warning_logs_but_allows_normal_finish(caplog, monkeypatch):
url = "data:,"
settings = Settings(
{
"MEMUSAGE_ENABLED": True,
"MEMUSAGE_WARNING_MB": 50,
"MEMUSAGE_LIMIT_MB": 0, # no hard limit
"MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01,
"LOG_LEVEL": "INFO",
}
)
# Respect the reactor already installed by the test env/CI.
settings.set("TWISTED_REACTOR", _installed_reactor_path(), priority="cmdline")
settings = {
"MEMUSAGE_ENABLED": True,
"MEMUSAGE_WARNING_MB": 50,
"MEMUSAGE_LIMIT_MB": 0, # no hard limit
"MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01,
"LOG_LEVEL": "INFO",
}
monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MiB)
MB = 1024 * 1024
monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB)
crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)