mirror of https://github.com/scrapy/scrapy.git
Revert "tests(memusage): make integration tests synchronous to avoid stray logs"
This reverts commit f086940b0a.
This commit is contained in:
parent
f086940b0a
commit
1fdd5e71b9
|
|
@ -4,10 +4,10 @@ import logging
|
|||
import sys
|
||||
|
||||
import pytest
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.extensions import memusage as memusage_mod
|
||||
from scrapy.extensions.memusage import MemoryUsage
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
|
@ -36,18 +36,21 @@ class _LoopSpider(Spider):
|
|||
)
|
||||
|
||||
|
||||
def _engine_started_once(self: MemoryUsage):
|
||||
"""
|
||||
Test-only replacement for MemoryUsage.engine_started:
|
||||
run checks once, synchronously; do not schedule periodic LoopingCalls.
|
||||
"""
|
||||
# keep side-effects identical to a single immediate tick
|
||||
self.update()
|
||||
if self.limit:
|
||||
self._check_limit()
|
||||
if self.warning:
|
||||
self._check_warning()
|
||||
return defer.succeed(None)
|
||||
class _OneShotLoop:
|
||||
"""Test stub for create_looping_call: run once immediately, no background task."""
|
||||
|
||||
def __init__(self, func):
|
||||
self.func = func
|
||||
self.running = False
|
||||
|
||||
def start(self, _interval, now: bool = False, **_kw):
|
||||
self.running = True
|
||||
if now:
|
||||
self.func()
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self.running = False
|
||||
|
||||
|
||||
@inlineCallbacks
|
||||
|
|
@ -60,10 +63,8 @@ def test_memusage_limit_closes_spider_with_reason_and_error_log(caplog, monkeypa
|
|||
"LOG_LEVEL": "INFO",
|
||||
}
|
||||
|
||||
# Avoid background timers; run memusage checks once synchronously.
|
||||
monkeypatch.setattr(
|
||||
MemoryUsage, "engine_started", _engine_started_once, raising=True
|
||||
)
|
||||
# Avoid background LoopingCall that can log after the test finishes.
|
||||
monkeypatch.setattr(memusage_mod, "create_looping_call", lambda f: _OneShotLoop(f))
|
||||
|
||||
MB = 1024 * 1024
|
||||
state = {"high": False}
|
||||
|
|
@ -71,7 +72,7 @@ def test_memusage_limit_closes_spider_with_reason_and_error_log(caplog, monkeypa
|
|||
def fake_vsz(self):
|
||||
return 250 * MB if state["high"] else 5 * MB
|
||||
|
||||
monkeypatch.setattr(MemoryUsage, "get_virtual_size", fake_vsz, raising=True)
|
||||
monkeypatch.setattr(MemoryUsage, "get_virtual_size", fake_vsz)
|
||||
|
||||
crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)
|
||||
|
||||
|
|
@ -100,15 +101,11 @@ def test_memusage_warning_logs_but_allows_normal_finish(caplog, monkeypatch):
|
|||
"LOG_LEVEL": "INFO",
|
||||
}
|
||||
|
||||
# Avoid background timers; run memusage checks once synchronously.
|
||||
monkeypatch.setattr(
|
||||
MemoryUsage, "engine_started", _engine_started_once, raising=True
|
||||
)
|
||||
# Avoid background LoopingCall that can log after the test finishes.
|
||||
monkeypatch.setattr(memusage_mod, "create_looping_call", lambda f: _OneShotLoop(f))
|
||||
|
||||
MB = 1024 * 1024
|
||||
monkeypatch.setattr(
|
||||
MemoryUsage, "get_virtual_size", lambda self: 75 * MB, raising=True
|
||||
)
|
||||
monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB)
|
||||
|
||||
crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue