[Solved] JOBDIR= None for when Scheduler initializes disk queue even if JOBDIR is empty string (#6124)

Co-authored-by: John Doe <johndoe@email.com>

docs/topics/settings.rst

@@ -1122,7 +1122,7 @@ modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
 JOBDIR
 ------
 
-Default: ``''``
+Default: ``None``
 
 A string indicating the directory for storing the state of a crawl when
 :ref:`pausing and resuming crawls <topics-jobs>`.

scrapy/core/scheduler.py

@@ -352,7 +352,7 @@ class Scheduler(BaseScheduler):
 
     def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
         """Return a folder name to keep disk queue state at"""
-        if jobdir is not None:
+        if jobdir:
             dqdir = Path(jobdir, "requests.queue")
             if not dqdir.exists():
                 dqdir.mkdir(parents=True)

scrapy/settings/default_settings.py

@@ -206,6 +206,8 @@ ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"
 ITEM_PIPELINES = {}
 ITEM_PIPELINES_BASE = {}
 
+JOBDIR = None
+
 LOG_ENABLED = True
 LOG_ENCODING = "utf-8"
 LOG_FORMATTER = "scrapy.logformatter.LogFormatter"

scrapy/utils/job.py

@@ -5,7 +5,9 @@ from scrapy.settings import BaseSettings
 
 
 def job_dir(settings: BaseSettings) -> Optional[str]:
-    path: str = settings["JOBDIR"]
-    if path and not Path(path).exists():
+    path: Optional[str] = settings["JOBDIR"]
+    if not path:
+        return None
+    if not Path(path).exists():
         Path(path).mkdir(parents=True)
     return path