scrapy/tests/test_pipeline_media.py

from __future__ import annotations

from unittest.mock import MagicMock

import pytest
from testfixtures import LogCapture
from twisted.python.failure import Failure

from scrapy import signals
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException
from scrapy.pipelines.media import MediaPipeline
from scrapy.utils.defer import _defer_sleep_async
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test


async def _mocked_download_func(request):
    assert request.callback is NO_CALLBACK
    response = request.meta.get("response")
    if callable(response):
        response = await response()
    if isinstance(response, Exception):
        raise response
    return response


class UserDefinedPipeline(MediaPipeline):
    def media_to_download(self, request, info, *, item=None):
        pass

    def get_media_requests(self, item, info):
        pass

    def media_downloaded(self, response, request, info, *, item=None):
        return {}

    def media_failed(self, failure, request, info):
        failure.raiseException()

    def file_path(self, request, response=None, info=None, *, item=None):
        return ""


class TestBaseMediaPipeline:
    pipeline_class = UserDefinedPipeline
    settings = None

    def setup_method(self):
        crawler = get_crawler(DefaultSpider, self.settings)
        crawler.spider = crawler._create_spider()
        crawler.engine = MagicMock(download_async=_mocked_download_func)
        self.pipe = self.pipeline_class.from_crawler(crawler)
        self.pipe.open_spider()
        self.info = self.pipe.spiderinfo
        self.fingerprint = crawler.request_fingerprinter.fingerprint

    def teardown_method(self):
        for name, signal in vars(signals).items():
            if not name.startswith("_"):
                disconnect_all(signal)

    def test_modify_media_request(self):
        request = Request("http://url")
        self.pipe._modify_media_request(request)
        assert request.meta == {"handle_httpstatus_all": True}

    def test_should_remove_req_res_references_before_caching_the_results(self):
        """Regression test case to prevent a memory leak in the Media Pipeline.

        The memory leak is triggered when an exception is raised when a Response
        scheduled by the Media Pipeline is being returned. For example, when a
        FileException('download-error') is raised because the Response status
        code is not 200 OK.

        It happens because we are keeping a reference to the Response object
        inside the FileException context. This is caused by the way Twisted
        return values from inline callbacks. It raises a custom exception
        encapsulating the original return value.

        The solution is to remove the exception context when this context is a
        _DefGen_Return instance, the BaseException used by Twisted to pass the
        returned value from those inline callbacks.

        Maybe there's a better and more reliable way to test the case described
        here, but it would be more complicated and involve running - or at least
        mocking - some async steps from the Media Pipeline. The current test
        case is simple and detects the problem very fast. On the other hand, it
        would not detect another kind of leak happening due to old object
        references being kept inside the Media Pipeline cache.

        This problem does not occur in Python 2.7 since we don't have Exception
        Chaining (https://www.python.org/dev/peps/pep-3134/).
        """
        # Create sample pair of Request and Response objects
        request = Request("http://url")
        response = Response("http://url", body=b"", request=request)

        # Simulate the Media Pipeline behavior to produce a Twisted Failure
        try:
            # Simulate a Twisted inline callback returning a Response
            raise StopIteration(response)
        except StopIteration as exc:
            def_gen_return_exc = exc
            try:
                # Simulate the media_downloaded callback raising a FileException
                # This usually happens when the status code is not 200 OK
                raise FileException("download-error")
            except Exception as exc:
                file_exc = exc
                # Simulate Twisted capturing the FileException
                # It encapsulates the exception inside a Twisted Failure
                failure = Failure(file_exc)

        # The Failure should encapsulate a FileException ...
        assert failure.value == file_exc
        # ... and it should have the StopIteration exception set as its context
        assert failure.value.__context__ == def_gen_return_exc

        # Let's calculate the request fingerprint and fake some runtime data...
        fp = self.fingerprint(request)
        info = self.pipe.spiderinfo
        info.downloading.add(fp)
        info.waiting[fp] = []

        # When calling the method that caches the Request's result ...
        self.pipe._cache_result_and_execute_waiters(failure, fp, info)
        # ... it should store the Twisted Failure ...
        assert info.downloaded[fp] == failure
        # ... encapsulating the original FileException ...
        assert info.downloaded[fp].value == file_exc
        # ... but it should not store the StopIteration exception on its context
        context = getattr(info.downloaded[fp].value, "__context__", None)
        assert context is None

    def test_default_item_completed(self):
        item = {"name": "name"}
        assert self.pipe.item_completed([], item, self.info) is item

        # Check that failures are logged by default
        fail = Failure(Exception())
        results = [(True, 1), (False, fail)]

        with LogCapture() as log:
            new_item = self.pipe.item_completed(results, item, self.info)

        assert new_item is item
        assert len(log.records) == 1
        record = log.records[0]
        assert record.levelname == "ERROR"
        assert record.exc_info == failure_to_exc_info(fail)

        # disable failure logging and check again
        self.pipe.LOG_FAILED_RESULTS = False
        with LogCapture() as log:
            new_item = self.pipe.item_completed(results, item, self.info)
        assert new_item is item
        assert len(log.records) == 0

    @coroutine_test
    async def test_default_process_item(self):
        item = {"name": "name"}
        new_item = await self.pipe.process_item(item)
        assert new_item is item


class MockedMediaPipeline(UserDefinedPipeline):
    def __init__(self, *args, crawler=None, **kwargs):
        super().__init__(*args, crawler=crawler, **kwargs)
        self._mockcalled = []

    def media_to_download(self, request, info, *, item=None):
        self._mockcalled.append("media_to_download")
        if "result" in request.meta:
            return request.meta.get("result")
        return super().media_to_download(request, info)

    def get_media_requests(self, item, info):
        self._mockcalled.append("get_media_requests")
        return item.get("requests")

    def media_downloaded(self, response, request, info, *, item=None):
        self._mockcalled.append("media_downloaded")
        return super().media_downloaded(response, request, info)

    def media_failed(self, failure, request, info):
        self._mockcalled.append("media_failed")
        return super().media_failed(failure, request, info)

    def item_completed(self, results, item, info):
        self._mockcalled.append("item_completed")
        item = super().item_completed(results, item, info)
        item["results"] = results
        return item


class AsyncMediaDownloadedPipeline(MockedMediaPipeline):
    async def media_downloaded(self, response, request, info, *, item=None):
        return super().media_downloaded(response, request, info)


class TestMediaPipeline(TestBaseMediaPipeline):
    pipeline_class = MockedMediaPipeline

    def _errback(self, result):
        self.pipe._mockcalled.append("request_errback")
        return result

    @coroutine_test
    async def test_result_succeed(self):
        rsp = Response("http://url1")
        req = Request(
            "http://url1",
            meta={"response": rsp},
            errback=self._errback,
        )
        item = {"requests": req}
        new_item = await self.pipe.process_item(item)
        assert new_item["results"] == [(True, {})]
        assert self.pipe._mockcalled == [
            "get_media_requests",
            "media_to_download",
            "media_downloaded",
            "item_completed",
        ]

    @coroutine_test
    async def test_result_failure(self):
        self.pipe.LOG_FAILED_RESULTS = False
        exc = Exception("foo")
        req = Request(
            "http://url1",
            meta={"response": exc},
            errback=self._errback,
        )
        item = {"requests": req}
        new_item = await self.pipe.process_item(item)
        assert len(new_item["results"]) == 1
        assert new_item["results"][0][0] is False
        assert isinstance(new_item["results"][0][1], Failure)
        assert new_item["results"][0][1].value == exc
        assert self.pipe._mockcalled == [
            "get_media_requests",
            "media_to_download",
            "media_failed",
            "request_errback",
            "item_completed",
        ]

    @coroutine_test
    async def test_mix_of_success_and_failure(self):
        self.pipe.LOG_FAILED_RESULTS = False
        rsp1 = Response("http://url1")
        req1 = Request("http://url1", meta={"response": rsp1})
        exc = Exception("foo")
        req2 = Request("http://url2", meta={"response": exc})
        item = {"requests": [req1, req2]}
        new_item = await self.pipe.process_item(item)
        assert len(new_item["results"]) == 2
        assert new_item["results"][0] == (True, {})
        assert new_item["results"][1][0] is False
        assert isinstance(new_item["results"][1][1], Failure)
        assert new_item["results"][1][1].value == exc
        m = self.pipe._mockcalled
        # only once
        assert m[0] == "get_media_requests"  # first hook called
        assert m.count("get_media_requests") == 1
        assert m.count("item_completed") == 1
        assert m[-1] == "item_completed"  # last hook called
        # twice, one per request
        assert m.count("media_to_download") == 2
        # one to handle success and other for failure
        assert m.count("media_downloaded") == 1
        assert m.count("media_failed") == 1

    @coroutine_test
    async def test_get_media_requests(self):
        # returns single Request (without callback)
        req = Request("http://url")
        item = {"requests": req}  # pass a single item
        new_item = await self.pipe.process_item(item)
        assert new_item is item
        assert self.fingerprint(req) in self.info.downloaded

        # returns iterable of Requests
        req1 = Request("http://url1")
        req2 = Request("http://url2")
        item = {"requests": iter([req1, req2])}
        new_item = await self.pipe.process_item(item)
        assert new_item is item
        assert self.fingerprint(req1) in self.info.downloaded
        assert self.fingerprint(req2) in self.info.downloaded

    @coroutine_test
    async def test_results_are_cached_across_multiple_items(self):
        rsp1 = Response("http://url1")
        req1 = Request("http://url1", meta={"response": rsp1})
        item = {"requests": req1}
        new_item = await self.pipe.process_item(item)
        assert new_item is item
        assert new_item["results"] == [(True, {})]

        # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same
        req2 = Request(
            req1.url, meta={"response": Response("http://donot.download.me")}
        )
        item = {"requests": req2}
        new_item = await self.pipe.process_item(item)
        assert new_item is item
        assert self.fingerprint(req1) == self.fingerprint(req2)
        assert new_item["results"] == [(True, {})]

    @coroutine_test
    async def test_results_are_cached_for_requests_of_single_item(self):
        rsp1 = Response("http://url1")
        req1 = Request("http://url1", meta={"response": rsp1})
        req2 = Request(
            req1.url, meta={"response": Response("http://donot.download.me")}
        )
        item = {"requests": [req1, req2]}
        new_item = await self.pipe.process_item(item)
        assert new_item is item
        assert new_item["results"] == [(True, {}), (True, {})]

    @coroutine_test
    async def test_wait_if_request_is_downloading(self):
        def _check_downloading(response):
            fp = self.fingerprint(req1)
            assert fp in self.info.downloading
            assert fp in self.info.waiting
            assert fp not in self.info.downloaded
            assert len(self.info.waiting[fp]) == 2
            return response

        rsp1 = Response("http://url")

        async def rsp1_func():
            await _defer_sleep_async()
            _check_downloading(rsp1)

        async def rsp2_func():
            pytest.fail("it must cache rsp1 result and must not try to redownload")

        req1 = Request("http://url", meta={"response": rsp1_func})
        req2 = Request(req1.url, meta={"response": rsp2_func})
        item = {"requests": [req1, req2]}
        new_item = await self.pipe.process_item(item)
        assert new_item["results"] == [(True, {}), (True, {})]

    @coroutine_test
    async def test_use_media_to_download_result(self):
        req = Request("http://url", meta={"result": "ITSME"})
        item = {"requests": req}
        new_item = await self.pipe.process_item(item)
        assert new_item["results"] == [(True, "ITSME")]
        assert self.pipe._mockcalled == [
            "get_media_requests",
            "media_to_download",
            "item_completed",
        ]

    def test_key_for_pipe(self):
        assert (
            self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
            == "MOCKEDMEDIAPIPELINE_IMAGES"
        )


class TestAsyncMediaDownloaded(TestMediaPipeline):
    pipeline_class = AsyncMediaDownloadedPipeline

    def test_key_for_pipe(self):
        assert (
            self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
            == "ASYNCMEDIADOWNLOADEDPIPELINE_IMAGES"
        )


class TestMediaPipelineAllowRedirectSettings:
    def _assert_request_no3xx(self, pipeline_class, settings):
        pipe = pipeline_class(crawler=get_crawler(None, settings))
        request = Request("http://url")
        pipe._modify_media_request(request)

        assert "handle_httpstatus_list" in request.meta
        for status, check in [
            (200, True),
            # These are the status codes we want
            # the downloader to handle itself
            (301, False),
            (302, False),
            (302, False),
            (307, False),
            (308, False),
            # we still want to get 4xx and 5xx
            (400, True),
            (404, True),
            (500, True),
        ]:
            if check:
                assert status in request.meta["handle_httpstatus_list"]
            else:
                assert status not in request.meta["handle_httpstatus_list"]

    def test_subclass_standard_setting(self):
        self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True})

    def test_subclass_specific_setting(self):
        self._assert_request_no3xx(
            UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True}
        )


class TestBuildFromCrawler:
    def setup_method(self):
        self.crawler = get_crawler(None, {"FILES_STORE": "/foo"})

    def test_simple(self):
        class Pipeline(UserDefinedPipeline):
            pass

        pipe = Pipeline.from_crawler(self.crawler)
        assert pipe.crawler == self.crawler
        assert pipe._fingerprinter

    def test_has_from_crawler_and_init(self):
        class Pipeline(UserDefinedPipeline):
            _from_crawler_called = False

            def __init__(self, store_uri, settings, *, crawler):
                super().__init__(crawler=crawler)
                self._init_called = True

            @classmethod
            def from_crawler(cls, crawler):
                settings = crawler.settings
                store_uri = settings["FILES_STORE"]
                o = cls(store_uri, settings=settings, crawler=crawler)
                o._from_crawler_called = True
                return o

        pipe = Pipeline.from_crawler(self.crawler)
        assert pipe.crawler == self.crawler
        assert pipe._fingerprinter
        assert pipe._from_crawler_called
        assert pipe._init_called

    def test_has_from_crawler(self):
        class Pipeline(UserDefinedPipeline):
            _from_crawler_called = False

            @classmethod
            def from_crawler(cls, crawler):
                settings = crawler.settings
                o = super().from_crawler(crawler)
                o._from_crawler_called = True
                o.store_uri = settings["FILES_STORE"]
                return o

        pipe = Pipeline.from_crawler(self.crawler)
        assert pipe.crawler == self.crawler
        assert pipe._fingerprinter
        assert pipe._from_crawler_called


class MediaFailedFailurePipeline(MockedMediaPipeline):
    def media_failed(self, failure, request, info):
        self._mockcalled.append("media_failed")
        return failure  # deprecated


class TestMediaFailedFailure(TestBaseMediaPipeline):
    """Test that media_failed() can return a failure instead of raising."""

    pipeline_class = MediaFailedFailurePipeline

    def _errback(self, result):
        self.pipe._mockcalled.append("request_errback")
        return result

    @coroutine_test
    async def test_result_failure(self):
        self.pipe.LOG_FAILED_RESULTS = False
        exc = Exception("foo")
        req = Request(
            "http://url1",
            meta={"response": exc},
            errback=self._errback,
        )
        item = {"requests": req}
        with pytest.warns(
            ScrapyDeprecationWarning, match="media_failed returned a Failure instance"
        ):
            new_item = await self.pipe.process_item(item)
        assert len(new_item["results"]) == 1
        assert new_item["results"][0][0] is False
        assert isinstance(new_item["results"][0][1], Failure)
        assert new_item["results"][0][1].value == exc
        assert self.pipe._mockcalled == [
            "get_media_requests",
            "media_to_download",
            "media_failed",
            "request_errback",
            "item_completed",
        ]