diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index d5ad59346..6a054ded5 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,17 +1,19 @@ from __future__ import annotations import logging +from typing import TYPE_CHECKING import pytest -from testfixtures import LogCapture from scrapy.http import Request, Response from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.spiders import MockServerSpider -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer class _HttpErrorSpider(MockServerSpider): @@ -192,65 +194,66 @@ class TestHttpErrorMiddlewareHandleAll: class TestHttpErrorMiddlewareIntegrational: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - @inline_callbacks_test - def test_middleware_works(self): + @coroutine_test + async def test_middleware_works(self, mockserver: MockServer) -> None: crawler = get_crawler(_HttpErrorSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) assert not crawler.spider.skipped assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} + assert crawler.stats get_value = crawler.stats.get_value assert get_value("httperror/response_ignored_count") == 3 assert get_value("httperror/response_ignored_status_count/404") == 1 assert get_value("httperror/response_ignored_status_count/402") == 1 assert get_value("httperror/response_ignored_status_count/500") == 1 - @inline_callbacks_test - def test_logging(self): + @coroutine_test + async def test_logging( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(_HttpErrorSpider) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver, bypass_status_codes={402}) + assert isinstance(crawler.spider, _HttpErrorSpider) assert crawler.spider.parsed == {"200", "402"} assert crawler.spider.skipped == {"402"} assert crawler.spider.failed == {"404", "500"} - assert "Ignoring response <404" in str(log) - assert "Ignoring response <500" in str(log) - assert "Ignoring response <200" not in str(log) - assert "Ignoring response <402" not in str(log) + assert "Ignoring response <404" in caplog.text + assert "Ignoring response <500" in caplog.text + assert "Ignoring response <200" not in caplog.text + assert "Ignoring response <402" not in caplog.text - @inline_callbacks_test - def test_logging_level(self): + @coroutine_test + async def test_logging_level( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # HttpError logs ignored responses with level INFO crawler = get_crawler(_HttpErrorSpider) - with LogCapture(level=logging.INFO) as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} - assert "Ignoring response <402" in str(log) - assert "Ignoring response <404" in str(log) - assert "Ignoring response <500" in str(log) - assert "Ignoring response <200" not in str(log) + assert "Ignoring response <402" in caplog.text + assert "Ignoring response <404" in caplog.text + assert "Ignoring response <500" in caplog.text + assert "Ignoring response <200" not in caplog.text # with level WARNING, we shouldn't capture anything from HttpError + caplog.clear() crawler = get_crawler(_HttpErrorSpider) - with LogCapture(level=logging.WARNING) as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.WARNING): + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} - assert "Ignoring response <402" not in str(log) - assert "Ignoring response <404" not in str(log) - assert "Ignoring response <500" not in str(log) - assert "Ignoring response <200" not in str(log) + assert "Ignoring response <402" not in caplog.text + assert "Ignoring response <404" not in caplog.text + assert "Ignoring response <500" not in caplog.text + assert "Ignoring response <200" not in caplog.text diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 318370d40..8e5020022 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -42,33 +42,29 @@ class TestFailureToExcInfo: class TestTopLevelFormatter: - def setup_method(self): - self.handler = LogCapture() - self.handler.addFilter(TopLevelFormatter(["test"])) - - def test_top_level_logger(self): + def test_top_level_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test") - with self.handler as log: - logger.warning("test log msg") - log.check(("test", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples - def test_children_logger(self): + def test_children_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test.test1") - with self.handler as log: - logger.warning("test log msg") - log.check(("test", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples - def test_overlapping_name_logger(self): + def test_overlapping_name_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test2") - with self.handler as log: - logger.warning("test log msg") - log.check(("test2", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test2", logging.WARNING, "test log msg") in caplog.record_tuples - def test_different_name_logger(self): + def test_different_name_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("different") - with self.handler as log: - logger.warning("test log msg") - log.check(("different", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("different", logging.WARNING, "test log msg") in caplog.record_tuples class TestLogCounterHandler: