Clean up setting getter defaults (#6892)

scrapy/core/downloader/__init__.py

@@ -124,7 +124,7 @@ class Downloader:
         )
         self._slot_gc_loop.start(60)
         self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict(
-            "DOWNLOAD_SLOTS", {}
+            "DOWNLOAD_SLOTS"
         )
 
     @inlineCallbacks

scrapy/downloadermiddlewares/ajaxcrawl.py

@@ -43,7 +43,7 @@ class AjaxCrawlMiddleware:
         # middleware parses first 4k. 4k turns out to be insufficient
         # for this middleware, and parsing 100k could be slow.
         # We use something in between (32K) by default.
-        self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768)
+        self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE")
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:

scrapy/downloadermiddlewares/robotstxt.py

@@ -38,10 +38,8 @@ class RobotsTxtMiddleware:
     def __init__(self, crawler: Crawler):
         if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
             raise NotConfigured
-        self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy")
-        self._robotstxt_useragent: str | None = crawler.settings.get(
-            "ROBOTSTXT_USER_AGENT", None
-        )
+        self._default_useragent: str = crawler.settings["USER_AGENT"]
+        self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"]
         self.crawler: Crawler = crawler
         self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {}
         self._parserimpl: RobotParser = load_object(

scrapy/extensions/feedexport.py

@@ -479,7 +479,7 @@ class FeedExporter:
             uri = self.settings["FEED_URI"]
             # handle pathlib.Path objects
             uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
-            feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}
+            feed_options = {"format": self.settings["FEED_FORMAT"]}
             self.feeds[uri] = feed_complete_default_values_from_settings(
                 feed_options, self.settings
             )

scrapy/extensions/periodic_log.py

@@ -78,7 +78,7 @@ class PeriodicLog:
             )
 
         ext_timing_enabled: bool = crawler.settings.getbool(
-            "PERIODIC_LOG_TIMING_ENABLED", False
+            "PERIODIC_LOG_TIMING_ENABLED"
         )
         if not (ext_stats or ext_delta or ext_timing_enabled):
             raise NotConfigured

scrapy/settings/default_settings.py

@@ -20,6 +20,7 @@ from pathlib import Path
 ADDONS = {}
 
 AJAXCRAWL_ENABLED = False
+AJAXCRAWL_MAXSIZE = 32768
 
 ASYNCIO_EVENT_LOOP = None
 
@@ -49,6 +50,8 @@ CONCURRENT_REQUESTS_PER_IP = 0
 COOKIES_ENABLED = True
 COOKIES_DEBUG = False
 
+CRAWLSPIDER_FOLLOW_LINKS = True
+
 DEFAULT_DROPITEM_LOG_LEVEL = "WARNING"
 
 DEFAULT_ITEM_CLASS = "scrapy.item.Item"
@@ -158,6 +161,7 @@ FEED_EXPORTERS_BASE = {
     "marshal": "scrapy.exporters.MarshalItemExporter",
     "pickle": "scrapy.exporters.PickleItemExporter",
 }
+FEED_FORMAT = "jsonlines"
 FEED_STORE_EMPTY = True
 FEED_STORAGES = {}
 FEED_STORAGES_BASE = {

scrapy/spiders/crawl.py

@@ -213,7 +213,5 @@ class CrawlSpider(Spider):
     @classmethod
     def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
         spider = super().from_crawler(crawler, *args, **kwargs)
-        spider._follow_links = crawler.settings.getbool(
-            "CRAWLSPIDER_FOLLOW_LINKS", True
-        )
+        spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS")
         return spider