From c564d6c6107e2a6156a311afcbdb7ff19b25fb53 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Sun, 9 Aug 2026 01:30:34 +0200 Subject: [PATCH] Complete the work --- docs/topics/media-pipeline.rst | 7 ++++-- scrapy/pipelines/images.py | 2 +- tests/test_pipeline_files.py | 39 ++++++++++++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index b16066d0c..e9a55a13c 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -393,13 +393,16 @@ adjust this retention delay use the :setting:`FILES_EXPIRES` setting (or :setting:`IMAGES_EXPIRES`, in case of Images Pipeline), which specifies the delay in number of days: +.. versionchanged:: VERSION + Fractional numbers of days are now supported. + .. code-block:: python # 120 days of delay for files expiration FILES_EXPIRES = 120 - # 30 days of delay for images expiration - IMAGES_EXPIRES = 30 + # 12 hours of delay for images expiration + IMAGES_EXPIRES = 0.5 The default value for both settings is 90 days. diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 79b6c4f27..a774ca67d 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -93,7 +93,7 @@ class ImagesPipeline(FilesPipeline): base_class_name="ImagesPipeline", settings=settings, ) - self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES) + self.expires: float = settings.getfloat(resolve("IMAGES_EXPIRES"), self.EXPIRES) if not hasattr(self, "IMAGES_RESULT_FIELD"): self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELD diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4e7fb118b..792413133 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -95,8 +95,14 @@ class TestFilesPipeline: def teardown_method(self): rmtree(self.tempdir) - def _create_pipeline(self, pipeline_cls: type[FilesPipeline]) -> FilesPipeline: - crawler = get_crawler(DefaultSpider, {"FILES_STORE": self.tempdir}) + def _create_pipeline( + self, + pipeline_cls: type[FilesPipeline], + settings: dict[str, Any] | None = None, + ) -> FilesPipeline: + crawler = get_crawler( + DefaultSpider, {"FILES_STORE": self.tempdir, **(settings or {})} + ) crawler.spider = crawler._create_spider() crawler.engine = MagicMock(download_async=mocked_download_func) pipeline = pipeline_cls.from_crawler(crawler) @@ -228,6 +234,35 @@ class TestFilesPipeline: assert result["files"][0]["checksum"] != "abc" assert result["files"][0]["status"] == "downloaded" + @pytest.mark.parametrize( + ("age_hours", "expected_status"), [(6, "uptodate"), (18, "downloaded")] + ) + @coroutine_test + async def test_file_expiration_below_one_day( + self, age_hours: int, expected_status: str + ) -> None: + pipeline = self._create_pipeline(FilesPipeline, {"FILES_EXPIRES": 0.5}) + item_url = f"http://example.com/file-{age_hours}h.pdf" + item = _create_item_with_files(item_url) + with ( + mock.patch.object( + FSFilesStore, + "stat_file", + return_value={ + "checksum": "abc", + "last_modified": time.time() - age_hours * 60 * 60, + }, + ), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + mock.patch.object(FilesPipeline, "inc_stats", return_value=True), + ): + result = await pipeline.process_item(item) + assert result["files"][0]["status"] == expected_status + @coroutine_test async def test_file_cached(self): item_url = "http://example.com/file3.pdf"