from __future__ import annotations from unittest.mock import MagicMock import pytest from testfixtures import LogCapture from twisted.python.failure import Failure from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException from scrapy.pipelines.media import MediaPipeline from scrapy.utils.defer import _defer_sleep_async from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test async def _mocked_download_func(request): assert request.callback is NO_CALLBACK response = request.meta.get("response") if callable(response): response = await response() if isinstance(response, Exception): raise response return response class UserDefinedPipeline(MediaPipeline): def media_to_download(self, request, info, *, item=None): pass def get_media_requests(self, item, info): pass def media_downloaded(self, response, request, info, *, item=None): return {} def media_failed(self, failure, request, info): failure.raiseException() def file_path(self, request, response=None, info=None, *, item=None): return "" class TestBaseMediaPipeline: pipeline_class = UserDefinedPipeline settings = None def setup_method(self): crawler = get_crawler(DefaultSpider, self.settings) crawler.spider = crawler._create_spider() crawler.engine = MagicMock(download_async=_mocked_download_func) self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.open_spider() self.info = self.pipe.spiderinfo self.fingerprint = crawler.request_fingerprinter.fingerprint def teardown_method(self): for name, signal in vars(signals).items(): if not name.startswith("_"): disconnect_all(signal) def test_modify_media_request(self): request = Request("http://url") self.pipe._modify_media_request(request) assert request.meta == {"handle_httpstatus_all": True} def test_should_remove_req_res_references_before_caching_the_results(self): """Regression test case to prevent a memory leak in the Media Pipeline. The memory leak is triggered when an exception is raised when a Response scheduled by the Media Pipeline is being returned. For example, when a FileException('download-error') is raised because the Response status code is not 200 OK. It happens because we are keeping a reference to the Response object inside the FileException context. This is caused by the way Twisted return values from inline callbacks. It raises a custom exception encapsulating the original return value. The solution is to remove the exception context when this context is a _DefGen_Return instance, the BaseException used by Twisted to pass the returned value from those inline callbacks. Maybe there's a better and more reliable way to test the case described here, but it would be more complicated and involve running - or at least mocking - some async steps from the Media Pipeline. The current test case is simple and detects the problem very fast. On the other hand, it would not detect another kind of leak happening due to old object references being kept inside the Media Pipeline cache. This problem does not occur in Python 2.7 since we don't have Exception Chaining (https://www.python.org/dev/peps/pep-3134/). """ # Create sample pair of Request and Response objects request = Request("http://url") response = Response("http://url", body=b"", request=request) # Simulate the Media Pipeline behavior to produce a Twisted Failure try: # Simulate a Twisted inline callback returning a Response raise StopIteration(response) except StopIteration as exc: def_gen_return_exc = exc try: # Simulate the media_downloaded callback raising a FileException # This usually happens when the status code is not 200 OK raise FileException("download-error") except Exception as exc: file_exc = exc # Simulate Twisted capturing the FileException # It encapsulates the exception inside a Twisted Failure failure = Failure(file_exc) # The Failure should encapsulate a FileException ... assert failure.value == file_exc # ... and it should have the StopIteration exception set as its context assert failure.value.__context__ == def_gen_return_exc # Let's calculate the request fingerprint and fake some runtime data... fp = self.fingerprint(request) info = self.pipe.spiderinfo info.downloading.add(fp) info.waiting[fp] = [] # When calling the method that caches the Request's result ... self.pipe._cache_result_and_execute_waiters(failure, fp, info) # ... it should store the Twisted Failure ... assert info.downloaded[fp] == failure # ... encapsulating the original FileException ... assert info.downloaded[fp].value == file_exc # ... but it should not store the StopIteration exception on its context context = getattr(info.downloaded[fp].value, "__context__", None) assert context is None def test_default_item_completed(self): item = {"name": "name"} assert self.pipe.item_completed([], item, self.info) is item # Check that failures are logged by default fail = Failure(Exception()) results = [(True, 1), (False, fail)] with LogCapture() as log: new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item assert len(log.records) == 1 record = log.records[0] assert record.levelname == "ERROR" assert record.exc_info == failure_to_exc_info(fail) # disable failure logging and check again self.pipe.LOG_FAILED_RESULTS = False with LogCapture() as log: new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item assert len(log.records) == 0 @coroutine_test async def test_default_process_item(self): item = {"name": "name"} new_item = await self.pipe.process_item(item) assert new_item is item class MockedMediaPipeline(UserDefinedPipeline): def __init__(self, *args, crawler=None, **kwargs): super().__init__(*args, crawler=crawler, **kwargs) self._mockcalled = [] def media_to_download(self, request, info, *, item=None): self._mockcalled.append("media_to_download") if "result" in request.meta: return request.meta.get("result") return super().media_to_download(request, info) def get_media_requests(self, item, info): self._mockcalled.append("get_media_requests") return item.get("requests") def media_downloaded(self, response, request, info, *, item=None): self._mockcalled.append("media_downloaded") return super().media_downloaded(response, request, info) def media_failed(self, failure, request, info): self._mockcalled.append("media_failed") return super().media_failed(failure, request, info) def item_completed(self, results, item, info): self._mockcalled.append("item_completed") item = super().item_completed(results, item, info) item["results"] = results return item class AsyncMediaDownloadedPipeline(MockedMediaPipeline): async def media_downloaded(self, response, request, info, *, item=None): return super().media_downloaded(response, request, info) class TestMediaPipeline(TestBaseMediaPipeline): pipeline_class = MockedMediaPipeline def _errback(self, result): self.pipe._mockcalled.append("request_errback") return result @coroutine_test async def test_result_succeed(self): rsp = Response("http://url1") req = Request( "http://url1", meta={"response": rsp}, errback=self._errback, ) item = {"requests": req} new_item = await self.pipe.process_item(item) assert new_item["results"] == [(True, {})] assert self.pipe._mockcalled == [ "get_media_requests", "media_to_download", "media_downloaded", "item_completed", ] @coroutine_test async def test_result_failure(self): self.pipe.LOG_FAILED_RESULTS = False exc = Exception("foo") req = Request( "http://url1", meta={"response": exc}, errback=self._errback, ) item = {"requests": req} new_item = await self.pipe.process_item(item) assert len(new_item["results"]) == 1 assert new_item["results"][0][0] is False assert isinstance(new_item["results"][0][1], Failure) assert new_item["results"][0][1].value == exc assert self.pipe._mockcalled == [ "get_media_requests", "media_to_download", "media_failed", "request_errback", "item_completed", ] @coroutine_test async def test_mix_of_success_and_failure(self): self.pipe.LOG_FAILED_RESULTS = False rsp1 = Response("http://url1") req1 = Request("http://url1", meta={"response": rsp1}) exc = Exception("foo") req2 = Request("http://url2", meta={"response": exc}) item = {"requests": [req1, req2]} new_item = await self.pipe.process_item(item) assert len(new_item["results"]) == 2 assert new_item["results"][0] == (True, {}) assert new_item["results"][1][0] is False assert isinstance(new_item["results"][1][1], Failure) assert new_item["results"][1][1].value == exc m = self.pipe._mockcalled # only once assert m[0] == "get_media_requests" # first hook called assert m.count("get_media_requests") == 1 assert m.count("item_completed") == 1 assert m[-1] == "item_completed" # last hook called # twice, one per request assert m.count("media_to_download") == 2 # one to handle success and other for failure assert m.count("media_downloaded") == 1 assert m.count("media_failed") == 1 @coroutine_test async def test_get_media_requests(self): # returns single Request (without callback) req = Request("http://url") item = {"requests": req} # pass a single item new_item = await self.pipe.process_item(item) assert new_item is item assert self.fingerprint(req) in self.info.downloaded # returns iterable of Requests req1 = Request("http://url1") req2 = Request("http://url2") item = {"requests": iter([req1, req2])} new_item = await self.pipe.process_item(item) assert new_item is item assert self.fingerprint(req1) in self.info.downloaded assert self.fingerprint(req2) in self.info.downloaded @coroutine_test async def test_results_are_cached_across_multiple_items(self): rsp1 = Response("http://url1") req1 = Request("http://url1", meta={"response": rsp1}) item = {"requests": req1} new_item = await self.pipe.process_item(item) assert new_item is item assert new_item["results"] == [(True, {})] # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same req2 = Request( req1.url, meta={"response": Response("http://donot.download.me")} ) item = {"requests": req2} new_item = await self.pipe.process_item(item) assert new_item is item assert self.fingerprint(req1) == self.fingerprint(req2) assert new_item["results"] == [(True, {})] @coroutine_test async def test_results_are_cached_for_requests_of_single_item(self): rsp1 = Response("http://url1") req1 = Request("http://url1", meta={"response": rsp1}) req2 = Request( req1.url, meta={"response": Response("http://donot.download.me")} ) item = {"requests": [req1, req2]} new_item = await self.pipe.process_item(item) assert new_item is item assert new_item["results"] == [(True, {}), (True, {})] @coroutine_test async def test_wait_if_request_is_downloading(self): def _check_downloading(response): fp = self.fingerprint(req1) assert fp in self.info.downloading assert fp in self.info.waiting assert fp not in self.info.downloaded assert len(self.info.waiting[fp]) == 2 return response rsp1 = Response("http://url") async def rsp1_func(): await _defer_sleep_async() _check_downloading(rsp1) async def rsp2_func(): pytest.fail("it must cache rsp1 result and must not try to redownload") req1 = Request("http://url", meta={"response": rsp1_func}) req2 = Request(req1.url, meta={"response": rsp2_func}) item = {"requests": [req1, req2]} new_item = await self.pipe.process_item(item) assert new_item["results"] == [(True, {}), (True, {})] @coroutine_test async def test_use_media_to_download_result(self): req = Request("http://url", meta={"result": "ITSME"}) item = {"requests": req} new_item = await self.pipe.process_item(item) assert new_item["results"] == [(True, "ITSME")] assert self.pipe._mockcalled == [ "get_media_requests", "media_to_download", "item_completed", ] def test_key_for_pipe(self): assert ( self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline") == "MOCKEDMEDIAPIPELINE_IMAGES" ) class TestAsyncMediaDownloaded(TestMediaPipeline): pipeline_class = AsyncMediaDownloadedPipeline def test_key_for_pipe(self): assert ( self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline") == "ASYNCMEDIADOWNLOADEDPIPELINE_IMAGES" ) class TestMediaPipelineAllowRedirectSettings: def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(crawler=get_crawler(None, settings)) request = Request("http://url") pipe._modify_media_request(request) assert "handle_httpstatus_list" in request.meta for status, check in [ (200, True), # These are the status codes we want # the downloader to handle itself (301, False), (302, False), (302, False), (307, False), (308, False), # we still want to get 4xx and 5xx (400, True), (404, True), (500, True), ]: if check: assert status in request.meta["handle_httpstatus_list"] else: assert status not in request.meta["handle_httpstatus_list"] def test_subclass_standard_setting(self): self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) def test_subclass_specific_setting(self): self._assert_request_no3xx( UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True} ) class TestBuildFromCrawler: def setup_method(self): self.crawler = get_crawler(None, {"FILES_STORE": "/foo"}) def test_simple(self): class Pipeline(UserDefinedPipeline): pass pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter def test_has_from_crawler_and_init(self): class Pipeline(UserDefinedPipeline): _from_crawler_called = False def __init__(self, store_uri, settings, *, crawler): super().__init__(crawler=crawler) self._init_called = True @classmethod def from_crawler(cls, crawler): settings = crawler.settings store_uri = settings["FILES_STORE"] o = cls(store_uri, settings=settings, crawler=crawler) o._from_crawler_called = True return o pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter assert pipe._from_crawler_called assert pipe._init_called def test_has_from_crawler(self): class Pipeline(UserDefinedPipeline): _from_crawler_called = False @classmethod def from_crawler(cls, crawler): settings = crawler.settings o = super().from_crawler(crawler) o._from_crawler_called = True o.store_uri = settings["FILES_STORE"] return o pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter assert pipe._from_crawler_called class MediaFailedFailurePipeline(MockedMediaPipeline): def media_failed(self, failure, request, info): self._mockcalled.append("media_failed") return failure # deprecated class TestMediaFailedFailure(TestBaseMediaPipeline): """Test that media_failed() can return a failure instead of raising.""" pipeline_class = MediaFailedFailurePipeline def _errback(self, result): self.pipe._mockcalled.append("request_errback") return result @coroutine_test async def test_result_failure(self): self.pipe.LOG_FAILED_RESULTS = False exc = Exception("foo") req = Request( "http://url1", meta={"response": exc}, errback=self._errback, ) item = {"requests": req} with pytest.warns( ScrapyDeprecationWarning, match="media_failed returned a Failure instance" ): new_item = await self.pipe.process_item(item) assert len(new_item["results"]) == 1 assert new_item["results"][0][0] is False assert isinstance(new_item["results"][0][1], Failure) assert new_item["results"][0][1].value == exc assert self.pipe._mockcalled == [ "get_media_requests", "media_to_download", "media_failed", "request_errback", "item_completed", ]