From 85aeda365db01939f70d0888593e7808380c8514 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 16 Jun 2025 09:28:06 +0200
Subject: [PATCH] Clean up setting getter defaults (#6892)

---
 scrapy/core/downloader/__init__.py        | 2 +-
 scrapy/downloadermiddlewares/ajaxcrawl.py | 2 +-
 scrapy/downloadermiddlewares/robotstxt.py | 6 ++----
 scrapy/extensions/feedexport.py           | 2 +-
 scrapy/extensions/periodic_log.py         | 2 +-
 scrapy/settings/default_settings.py       | 4 ++++
 scrapy/spiders/crawl.py                   | 4 +---
 7 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 9293d7b78..4b5654826 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -124,7 +124,7 @@ class Downloader:
         )
         self._slot_gc_loop.start(60)
         self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict(
-            "DOWNLOAD_SLOTS", {}
+            "DOWNLOAD_SLOTS"
         )
 
     @inlineCallbacks
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index e7a8962a1..a23deaa45 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -43,7 +43,7 @@ class AjaxCrawlMiddleware:
         # middleware parses first 4k. 4k turns out to be insufficient
         # for this middleware, and parsing 100k could be slow.
         # We use something in between (32K) by default.
-        self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768)
+        self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE")
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index aba455bdd..fbd737970 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -38,10 +38,8 @@ class RobotsTxtMiddleware:
     def __init__(self, crawler: Crawler):
         if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
             raise NotConfigured
-        self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy")
-        self._robotstxt_useragent: str | None = crawler.settings.get(
-            "ROBOTSTXT_USER_AGENT", None
-        )
+        self._default_useragent: str = crawler.settings["USER_AGENT"]
+        self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"]
         self.crawler: Crawler = crawler
         self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {}
         self._parserimpl: RobotParser = load_object(
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c39a9c92e..d9e9ea775 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -479,7 +479,7 @@ class FeedExporter:
             uri = self.settings["FEED_URI"]
             # handle pathlib.Path objects
             uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
-            feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}
+            feed_options = {"format": self.settings["FEED_FORMAT"]}
             self.feeds[uri] = feed_complete_default_values_from_settings(
                 feed_options, self.settings
             )
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py
index 9158482fa..98210990a 100644
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@@ -78,7 +78,7 @@ class PeriodicLog:
             )
 
         ext_timing_enabled: bool = crawler.settings.getbool(
-            "PERIODIC_LOG_TIMING_ENABLED", False
+            "PERIODIC_LOG_TIMING_ENABLED"
         )
         if not (ext_stats or ext_delta or ext_timing_enabled):
             raise NotConfigured
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 7cd470f11..b6f47f1c3 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -20,6 +20,7 @@ from pathlib import Path
 ADDONS = {}
 
 AJAXCRAWL_ENABLED = False
+AJAXCRAWL_MAXSIZE = 32768
 
 ASYNCIO_EVENT_LOOP = None
 
@@ -49,6 +50,8 @@ CONCURRENT_REQUESTS_PER_IP = 0
 COOKIES_ENABLED = True
 COOKIES_DEBUG = False
 
+CRAWLSPIDER_FOLLOW_LINKS = True
+
 DEFAULT_DROPITEM_LOG_LEVEL = "WARNING"
 
 DEFAULT_ITEM_CLASS = "scrapy.item.Item"
@@ -158,6 +161,7 @@ FEED_EXPORTERS_BASE = {
     "marshal": "scrapy.exporters.MarshalItemExporter",
     "pickle": "scrapy.exporters.PickleItemExporter",
 }
+FEED_FORMAT = "jsonlines"
 FEED_STORE_EMPTY = True
 FEED_STORAGES = {}
 FEED_STORAGES_BASE = {
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index f44f70e40..98e7b23c0 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -213,7 +213,5 @@ class CrawlSpider(Spider):
     @classmethod
     def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
         spider = super().from_crawler(crawler, *args, **kwargs)
-        spider._follow_links = crawler.settings.getbool(
-            "CRAWLSPIDER_FOLLOW_LINKS", True
-        )
+        spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS")
         return spider