Revert "tests(memusage): make integration tests synchronous to avoid stray logs"

This reverts commit f086940b0a.
This commit is contained in:
Andrey Rakhmatullin 2026-06-18 19:44:29 +05:00
parent f086940b0a
commit 1fdd5e71b9
1 changed files with 22 additions and 25 deletions

View File

@ -4,10 +4,10 @@ import logging
import sys
import pytest
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from scrapy import signals
from scrapy.extensions import memusage as memusage_mod
from scrapy.extensions.memusage import MemoryUsage
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
@ -36,18 +36,21 @@ class _LoopSpider(Spider):
)
def _engine_started_once(self: MemoryUsage):
"""
Test-only replacement for MemoryUsage.engine_started:
run checks once, synchronously; do not schedule periodic LoopingCalls.
"""
# keep side-effects identical to a single immediate tick
self.update()
if self.limit:
self._check_limit()
if self.warning:
self._check_warning()
return defer.succeed(None)
class _OneShotLoop:
"""Test stub for create_looping_call: run once immediately, no background task."""
def __init__(self, func):
self.func = func
self.running = False
def start(self, _interval, now: bool = False, **_kw):
self.running = True
if now:
self.func()
return self
def stop(self):
self.running = False
@inlineCallbacks
@ -60,10 +63,8 @@ def test_memusage_limit_closes_spider_with_reason_and_error_log(caplog, monkeypa
"LOG_LEVEL": "INFO",
}
# Avoid background timers; run memusage checks once synchronously.
monkeypatch.setattr(
MemoryUsage, "engine_started", _engine_started_once, raising=True
)
# Avoid background LoopingCall that can log after the test finishes.
monkeypatch.setattr(memusage_mod, "create_looping_call", lambda f: _OneShotLoop(f))
MB = 1024 * 1024
state = {"high": False}
@ -71,7 +72,7 @@ def test_memusage_limit_closes_spider_with_reason_and_error_log(caplog, monkeypa
def fake_vsz(self):
return 250 * MB if state["high"] else 5 * MB
monkeypatch.setattr(MemoryUsage, "get_virtual_size", fake_vsz, raising=True)
monkeypatch.setattr(MemoryUsage, "get_virtual_size", fake_vsz)
crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)
@ -100,15 +101,11 @@ def test_memusage_warning_logs_but_allows_normal_finish(caplog, monkeypatch):
"LOG_LEVEL": "INFO",
}
# Avoid background timers; run memusage checks once synchronously.
monkeypatch.setattr(
MemoryUsage, "engine_started", _engine_started_once, raising=True
)
# Avoid background LoopingCall that can log after the test finishes.
monkeypatch.setattr(memusage_mod, "create_looping_call", lambda f: _OneShotLoop(f))
MB = 1024 * 1024
monkeypatch.setattr(
MemoryUsage, "get_virtual_size", lambda self: 75 * MB, raising=True
)
monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB)
crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)