diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 576feae7e..899bb81fd 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -424,13 +424,16 @@ adjust this retention delay use the :setting:`FILES_EXPIRES` setting (or :setting:`IMAGES_EXPIRES`, in case of Images Pipeline), which specifies the delay in number of days: +.. versionchanged:: VERSION + Fractional numbers of days are now supported. + .. code-block:: python # 120 days of delay for files expiration FILES_EXPIRES = 120 - # 30 days of delay for images expiration - IMAGES_EXPIRES = 30 + # 12 hours of delay for images expiration + IMAGES_EXPIRES = 0.5 The default value for both settings is 90 days. diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 9d67e98bf..78cd02c22 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -497,7 +497,7 @@ class FilesPipeline(MediaPipeline): resolve = functools.partial( self._key_for_pipe, base_class_name=cls_name, settings=settings ) - self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES) + self.expires: float = settings.getfloat(resolve("FILES_EXPIRES"), self.EXPIRES) if not hasattr(self, "FILES_URLS_FIELD"): self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD if not hasattr(self, "FILES_RESULT_FIELD"): diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 7186fc8de..430a6b927 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -88,7 +88,7 @@ class ImagesPipeline(FilesPipeline): base_class_name="ImagesPipeline", settings=settings, ) - self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES) + self.expires: float = settings.getfloat(resolve("IMAGES_EXPIRES"), self.EXPIRES) if not hasattr(self, "IMAGES_RESULT_FIELD"): self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELD diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 928f4e2c2..84c4e58ac 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -237,6 +237,35 @@ class TestFilesPipeline: assert result["files"][0]["checksum"] != "abc" assert result["files"][0]["status"] == "downloaded" + @pytest.mark.parametrize( + ("age_hours", "expected_status"), [(6, "uptodate"), (18, "downloaded")] + ) + @coroutine_test + async def test_file_expiration_below_one_day( + self, age_hours: int, expected_status: str + ) -> None: + pipeline = self._create_pipeline(FilesPipeline, {"FILES_EXPIRES": 0.5}) + item_url = f"http://example.com/file-{age_hours}h.pdf" + item = _create_item_with_files(item_url) + with ( + mock.patch.object( + FSFilesStore, + "stat_file", + return_value={ + "checksum": "abc", + "last_modified": time.time() - age_hours * 60 * 60, + }, + ), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + mock.patch.object(FilesPipeline, "inc_stats", return_value=True), + ): + result = await pipeline.process_item(item) + assert result["files"][0]["status"] == expected_status + @coroutine_test async def test_file_cached(self): item_url = "http://example.com/file3.pdf"