From b1dd893fbb4d2efed3342e6c98a6bbf4b393d48e Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 20 Jan 2023 02:17:02 -0800 Subject: [PATCH] Support Path Objects Issue #5739 --- scrapy/pipelines/files.py | 9 +++++---- tests/test_pipeline_files.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 51aedafe8..4b0c96b25 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -14,7 +14,7 @@ from contextlib import suppress from ftplib import FTP from io import BytesIO from pathlib import Path -from typing import DefaultDict, Optional, Set +from typing import DefaultDict, Optional, Set, Union from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -41,7 +41,8 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: str): + def __init__(self, basedir: Union[str, os.PathLike]): + basedir = str(basedir) # support Path object if '://' in basedir: basedir = basedir.split('://', 1)[1] self.basedir = basedir @@ -65,8 +66,8 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: str) -> Path: - path_comps = path.split('/') + def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: + path_comps = str(path).split('/') return Path(self.basedir, *path_comps) def _mkdir(self, dirname: Path, domain: Optional[str] = None): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4acd29bf7..5280ab0d2 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -405,6 +405,23 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value) + def test_file_pipeline_using_pathlike_objects(self): + + class CustomFilesPipelineWithPathLikeDir(FilesPipeline): + def file_path(self, request, response=None, info=None, *, item=None): + return Path('subdir') / Path(request.url).name + + pipeline = CustomFilesPipelineWithPathLikeDir.from_settings( + Settings({'FILES_STORE': Path('./Temp')}) + ) + request = Request("http://example.com/image01.jpg") + self.assertEqual(pipeline.file_path(request), Path('subdir/image01.jpg')) + + def test_files_store_constructor_with_pathlike_object(self): + path = Path('./FileDir') + fs_store = FSFilesStore(path) + self.assertEqual(fs_store.basedir, str(path)) + class TestS3FilesStore(unittest.TestCase):