From 84fb234cae1bffa944d9f5870c2fc87c75d35261 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Jan 2023 22:47:38 -0800 Subject: [PATCH] fixed additional trigger in FilesPipeline --- scrapy/pipelines/files.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4b0c96b25..a5948525a 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,13 +36,17 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) +def _to_string(path: Union[str, os.PathLike]): + return str(path) # convert a Path object to string + + class FileException(Exception): """General media error exception""" class FSFilesStore: def __init__(self, basedir: Union[str, os.PathLike]): - basedir = str(basedir) # support Path object + basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] self.basedir = basedir @@ -67,7 +71,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: - path_comps = str(path).split('/') + path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) def _mkdir(self, dirname: Path, domain: Optional[str] = None): @@ -323,12 +327,12 @@ class FilesPipeline(MediaPipeline): DEFAULT_FILES_RESULT_FIELD = 'files' def __init__(self, store_uri, download_func=None, settings=None): + store_uri = _to_string(store_uri) if not store_uri: raise NotConfigured if isinstance(settings, dict) or settings is None: settings = Settings(settings) - cls_name = "FilesPipeline" self.store = self._get_store(store_uri) resolve = functools.partial(self._key_for_pipe,