scrapy/tests/test_spidermiddleware_output_chain.py

from testfixtures import LogCapture

from scrapy import Request, Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import coroutine_test


class _BaseSpiderMiddleware:
    def __init__(self, crawler):
        self.crawler = crawler

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)


class LogExceptionMiddleware(_BaseSpiderMiddleware):
    def process_spider_exception(self, response, exception):
        self.crawler.spider.logger.info(
            "Middleware: %s exception caught", exception.__class__.__name__
        )


# ================================================================================
# (0) recover from an exception on a spider callback
class RecoveryMiddleware(_BaseSpiderMiddleware):
    def process_spider_exception(self, response, exception):
        self.crawler.spider.logger.info(
            "Middleware: %s exception caught", exception.__class__.__name__
        )
        return [
            {"from": "process_spider_exception"},
            Request(response.url, meta={"dont_fail": True}, dont_filter=True),
        ]


class RecoverySpider(Spider):
    name = "RecoverySpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES_BASE": {},
        "SPIDER_MIDDLEWARES": {
            RecoveryMiddleware: 10,
        },
    }

    async def start(self):
        yield Request(self.mockserver.url("/status?n=200"))

    def parse(self, response):
        yield {"test": 1}
        self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail"))
        if not response.meta.get("dont_fail"):
            raise TabError


class RecoveryAsyncGenSpider(RecoverySpider):
    name = "RecoveryAsyncGenSpider"

    async def parse(self, response):
        for r in super().parse(response):
            yield r


# ================================================================================
# (1) exceptions from a spider middleware's process_spider_input method
class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware):
    def process_spider_input(self, response):
        self.crawler.spider.logger.info("Middleware: will raise IndexError")
        raise IndexError


class ProcessSpiderInputSpiderWithoutErrback(Spider):
    name = "ProcessSpiderInputSpiderWithoutErrback"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            # spider
            FailProcessSpiderInputMiddleware: 8,
            LogExceptionMiddleware: 6,
            # engine
        }
    }

    async def start(self):
        yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse)

    def parse(self, response):
        return {"from": "callback"}


class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback):
    name = "ProcessSpiderInputSpiderWithErrback"

    async def start(self):
        yield Request(
            self.mockserver.url("/status?n=200"), self.parse, errback=self.errback
        )

    def errback(self, failure):
        self.logger.info("Got a Failure on the Request errback")
        return {"from": "errback"}


# ================================================================================
# (2) exceptions from a spider callback (generator)
class GeneratorCallbackSpider(Spider):
    name = "GeneratorCallbackSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            LogExceptionMiddleware: 10,
        },
    }

    async def start(self):
        yield Request(self.mockserver.url("/status?n=200"))

    def parse(self, response):
        yield {"test": 1}
        yield {"test": 2}
        raise ImportError


class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider):
    async def parse(self, response):
        yield {"test": 1}
        yield {"test": 2}
        raise ImportError


# ================================================================================
# (2.1) exceptions from a spider callback (generator, middleware right after callback)
class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider):
    name = "GeneratorCallbackSpiderMiddlewareRightAfterSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            LogExceptionMiddleware: 100000,
        },
    }


# ================================================================================
# (3) exceptions from a spider callback (not a generator)
class NotGeneratorCallbackSpider(Spider):
    name = "NotGeneratorCallbackSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            LogExceptionMiddleware: 10,
        },
    }

    async def start(self):
        yield Request(self.mockserver.url("/status?n=200"))

    def parse(self, response):
        return [{"test": 1}, {"test": 1 / 0}]


# ================================================================================
# (3.1) exceptions from a spider callback (not a generator, middleware right after callback)
class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackSpider):
    name = "NotGeneratorCallbackSpiderMiddlewareRightAfterSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            LogExceptionMiddleware: 100000,
        },
    }


# ================================================================================
# (4) exceptions from a middleware process_spider_output method (generator)
class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
    async def process_spider_output(self, response, result):
        async for r in result:
            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
            yield r

    def process_spider_exception(self, response, exception):
        method = f"{self.__class__.__name__}.process_spider_exception"
        self.crawler.spider.logger.info(
            "%s: %s caught", method, exception.__class__.__name__
        )


class GeneratorFailMiddleware(_BaseSpiderMiddleware):
    async def process_spider_output(self, response, result):
        async for r in result:
            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
            yield r
            raise LookupError

    def process_spider_exception(self, response, exception):
        method = f"{self.__class__.__name__}.process_spider_exception"
        self.crawler.spider.logger.info(
            "%s: %s caught", method, exception.__class__.__name__
        )
        yield {"processed": [method]}


class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
    pass


class GeneratorRecoverMiddleware(_BaseSpiderMiddleware):
    async def process_spider_output(self, response, result):
        async for r in result:
            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
            yield r

    def process_spider_exception(self, response, exception):
        method = f"{self.__class__.__name__}.process_spider_exception"
        self.crawler.spider.logger.info(
            "%s: %s caught", method, exception.__class__.__name__
        )
        yield {"processed": [method]}


class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
    pass


class GeneratorOutputChainSpider(Spider):
    name = "GeneratorOutputChainSpider"
    custom_settings = {
        "SPIDER_MIDDLEWARES": {
            GeneratorFailMiddleware: 10,
            GeneratorDoNothingAfterFailureMiddleware: 8,
            GeneratorRecoverMiddleware: 5,
            GeneratorDoNothingAfterRecoveryMiddleware: 3,
        },
    }

    async def start(self):
        yield Request(self.mockserver.url("/status?n=200"))

    def parse(self, response):
        yield {"processed": ["parse-first-item"]}
        yield {"processed": ["parse-second-item"]}


# ================================================================================
class TestSpiderMiddleware:
    mockserver: MockServer

    @classmethod
    def setup_class(cls):
        cls.mockserver = MockServer()
        cls.mockserver.__enter__()

    @classmethod
    def teardown_class(cls):
        cls.mockserver.__exit__(None, None, None)

    async def crawl_log(self, spider: type[Spider]) -> LogCapture:
        crawler = get_crawler(spider)
        with LogCapture() as log:
            await crawler.crawl_async(mockserver=self.mockserver)
        return log

    @coroutine_test
    async def test_recovery(self):
        """
        (0) Recover from an exception in a spider callback. The final item count should be 3
        (one yielded from the callback method before the exception is raised, one directly
        from the recovery middleware and one from the spider when processing the request that
        was enqueued from the recovery middleware)
        """
        log = await self.crawl_log(RecoverySpider)
        assert "Middleware: TabError exception caught" in str(log)
        assert str(log).count("Middleware: TabError exception caught") == 1
        assert "'item_scraped_count': 3" in str(log)

    @coroutine_test
    async def test_recovery_asyncgen(self):
        """
        Same as test_recovery but with an async callback.
        """
        log = await self.crawl_log(RecoveryAsyncGenSpider)
        assert "Middleware: TabError exception caught" in str(log)
        assert str(log).count("Middleware: TabError exception caught") == 1
        assert "'item_scraped_count': 3" in str(log)

    @coroutine_test
    async def test_process_spider_input_without_errback(self):
        """
        (1.1) An exception from the process_spider_input chain should be caught by the
        process_spider_exception chain from the start if the Request has no errback
        """
        log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)
        assert "Middleware: will raise IndexError" in str(log1)
        assert "Middleware: IndexError exception caught" in str(log1)

    @coroutine_test
    async def test_process_spider_input_with_errback(self):
        """
        (1.2) An exception from the process_spider_input chain should not be caught by the
        process_spider_exception chain if the Request has an errback
        """
        log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback)
        assert "Middleware: IndexError exception caught" not in str(log1)
        assert "Middleware: will raise IndexError" in str(log1)
        assert "Got a Failure on the Request errback" in str(log1)
        assert "{'from': 'errback'}" in str(log1)
        assert "{'from': 'callback'}" not in str(log1)
        assert "'item_scraped_count': 1" in str(log1)

    @coroutine_test
    async def test_generator_callback(self):
        """
        (2) An exception from a spider callback (returning a generator) should
        be caught by the process_spider_exception chain. Items yielded before the
        exception is raised should be processed normally.
        """
        log2 = await self.crawl_log(GeneratorCallbackSpider)
        assert "Middleware: ImportError exception caught" in str(log2)
        assert "'item_scraped_count': 2" in str(log2)

    @coroutine_test
    async def test_async_generator_callback(self):
        """
        Same as test_generator_callback but with an async callback.
        """
        log2 = await self.crawl_log(AsyncGeneratorCallbackSpider)
        assert "Middleware: ImportError exception caught" in str(log2)
        assert "'item_scraped_count': 2" in str(log2)

    @coroutine_test
    async def test_generator_callback_right_after_callback(self):
        """
        (2.1) Special case of (2): Exceptions should be caught
        even if the middleware is placed right after the spider
        """
        log21 = await self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider)
        assert "Middleware: ImportError exception caught" in str(log21)
        assert "'item_scraped_count': 2" in str(log21)

    @coroutine_test
    async def test_not_a_generator_callback(self):
        """
        (3) An exception from a spider callback (returning a list) should
        be caught by the process_spider_exception chain. No items should be processed.
        """
        log3 = await self.crawl_log(NotGeneratorCallbackSpider)
        assert "Middleware: ZeroDivisionError exception caught" in str(log3)
        assert "item_scraped_count" not in str(log3)

    @coroutine_test
    async def test_not_a_generator_callback_right_after_callback(self):
        """
        (3.1) Special case of (3): Exceptions should be caught
        even if the middleware is placed right after the spider
        """
        log31 = await self.crawl_log(
            NotGeneratorCallbackSpiderMiddlewareRightAfterSpider
        )
        assert "Middleware: ZeroDivisionError exception caught" in str(log31)
        assert "item_scraped_count" not in str(log31)

    @coroutine_test
    async def test_generator_output_chain(self):
        """
        (4) An exception from a middleware's process_spider_output method should be sent
        to the process_spider_exception method from the next middleware in the chain.
        The result of the recovery by the process_spider_exception method should be handled
        by the process_spider_output method from the next middleware.
        The final item count should be 2 (one from the spider callback and one from the
        process_spider_exception chain)
        """
        log4 = await self.crawl_log(GeneratorOutputChainSpider)
        assert "'item_scraped_count': 2" in str(log4)
        assert (
            "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught"
            in str(log4)
        )
        assert (
            "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught"
            in str(log4)
        )
        assert (
            "GeneratorFailMiddleware.process_spider_exception: LookupError caught"
            not in str(log4)
        )
        assert (
            "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught"
            not in str(log4)
        )
        item_from_callback = {
            "processed": [
                "parse-first-item",
                "GeneratorFailMiddleware.process_spider_output",
                "GeneratorDoNothingAfterFailureMiddleware.process_spider_output",
                "GeneratorRecoverMiddleware.process_spider_output",
                "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
            ]
        }
        item_recovered = {
            "processed": [
                "GeneratorRecoverMiddleware.process_spider_exception",
                "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
            ]
        }
        assert str(item_from_callback) in str(log4)
        assert str(item_recovered) in str(log4)
        assert "parse-second-item" not in str(log4)