Update default_settings, deprecate CRAWLSPIDER_FOLLOW_LINKS. (#7592)

docs/topics/media-pipeline.rst

@@ -81,9 +81,6 @@ thumbnailing and normalizing images to JPEG/RGB format.
 Enabling your Media Pipeline
 ============================
 
-.. setting:: IMAGES_STORE
-.. setting:: FILES_STORE
-
 To enable your media pipeline you must first add it to your project
 :setting:`ITEM_PIPELINES` setting.
 
@@ -102,6 +99,8 @@ For Files Pipeline, use:
 .. note::
     You can also use both the Files and Images Pipeline at the same time.
 
+.. setting:: IMAGES_STORE
+.. setting:: FILES_STORE
 
 Then, configure the target storage setting to a valid value that will be used
 for storing the downloaded images. Otherwise the pipeline will remain disabled,

scrapy/settings/default_settings.py

@@ -20,19 +20,26 @@ from typing import Any
 
 __all__ = [
     "ADDONS",
-    "AJAXCRAWL_ENABLED",
-    "AJAXCRAWL_MAXSIZE",
     "ASYNCIO_EVENT_LOOP",
     "AUTOTHROTTLE_DEBUG",
     "AUTOTHROTTLE_ENABLED",
     "AUTOTHROTTLE_MAX_DELAY",
     "AUTOTHROTTLE_START_DELAY",
     "AUTOTHROTTLE_TARGET_CONCURRENCY",
+    "AWS_ACCESS_KEY_ID",
+    "AWS_ENDPOINT_URL",
+    "AWS_REGION_NAME",
+    "AWS_SECRET_ACCESS_KEY",
+    "AWS_SESSION_TOKEN",
+    "AWS_USE_SSL",
+    "AWS_VERIFY",
     "BOT_NAME",
     "CLOSESPIDER_ERRORCOUNT",
     "CLOSESPIDER_ITEMCOUNT",
     "CLOSESPIDER_PAGECOUNT",
+    "CLOSESPIDER_PAGECOUNT_NO_ITEM",
     "CLOSESPIDER_TIMEOUT",
+    "CLOSESPIDER_TIMEOUT_NO_ITEM",
     "COMMANDS_MODULE",
     "COMPRESSION_ENABLED",
     "CONCURRENT_ITEMS",
@@ -64,11 +71,14 @@ __all__ = [
     "DOWNLOAD_HANDLERS",
     "DOWNLOAD_HANDLERS_BASE",
     "DOWNLOAD_MAXSIZE",
+    "DOWNLOAD_SLOTS",
     "DOWNLOAD_TIMEOUT",
     "DOWNLOAD_TLS_MAX_VERSION",
     "DOWNLOAD_TLS_MIN_VERSION",
+    "DOWNLOAD_VERIFY_CERTIFICATES",
     "DOWNLOAD_WARNSIZE",
     "DUPEFILTER_CLASS",
+    "DUPEFILTER_DEBUG",
     "EDITOR",
     "EXTENSIONS",
     "EXTENSIONS_BASE",
@@ -87,7 +97,9 @@ __all__ = [
     "FEED_STORAGE_S3_ACL",
     "FEED_STORE_EMPTY",
     "FEED_TEMPDIR",
+    "FEED_URI",
     "FEED_URI_PARAMS",
+    "FILES_STORE",
     "FILES_STORE_GCS_ACL",
     "FILES_STORE_S3_ACL",
     "FORCE_CRAWLER_PROCESS",
@@ -107,9 +119,12 @@ __all__ = [
     "HTTPCACHE_IGNORE_SCHEMES",
     "HTTPCACHE_POLICY",
     "HTTPCACHE_STORAGE",
+    "HTTPERROR_ALLOWED_CODES",
+    "HTTPERROR_ALLOW_ALL",
     "HTTPPROXY_AUTH_ENCODING",
     "HTTPPROXY_ENABLED",
     "HTTPX_HTTP2_ENABLED",
+    "IMAGES_STORE",
     "IMAGES_STORE_GCS_ACL",
     "IMAGES_STORE_S3_ACL",
     "ITEM_PIPELINES",
@@ -132,6 +147,8 @@ __all__ = [
     "MAIL_HOST",
     "MAIL_PASS",
     "MAIL_PORT",
+    "MAIL_SSL",
+    "MAIL_TLS",
     "MAIL_USER",
     "MEMDEBUG_ENABLED",
     "MEMDEBUG_NOTIFY",
@@ -153,6 +170,7 @@ __all__ = [
     "REDIRECT_MAX_TIMES",
     "REDIRECT_PRIORITY_ADJUST",
     "REFERER_ENABLED",
+    "REFERRER_POLICIES",
     "REFERRER_POLICY",
     "REQUEST_FINGERPRINTER_CLASS",
     "RETRY_ENABLED",
@@ -198,9 +216,6 @@ __all__ = [
 
 ADDONS = {}
 
-AJAXCRAWL_ENABLED = False
-AJAXCRAWL_MAXSIZE = 32768
-
 ASYNCIO_EVENT_LOOP = None
 
 AUTOTHROTTLE_ENABLED = False
@@ -209,12 +224,22 @@ AUTOTHROTTLE_MAX_DELAY = 60.0
 AUTOTHROTTLE_START_DELAY = 5.0
 AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
 
+AWS_ACCESS_KEY_ID = None
+AWS_SECRET_ACCESS_KEY = None
+AWS_ENDPOINT_URL = None
+AWS_REGION_NAME = None
+AWS_SESSION_TOKEN = None
+AWS_USE_SSL = None
+AWS_VERIFY = None
+
 BOT_NAME = "scrapybot"
 
 CLOSESPIDER_ERRORCOUNT = 0
 CLOSESPIDER_ITEMCOUNT = 0
 CLOSESPIDER_PAGECOUNT = 0
 CLOSESPIDER_TIMEOUT = 0
+CLOSESPIDER_PAGECOUNT_NO_ITEM = 0
+CLOSESPIDER_TIMEOUT_NO_ITEM = 0
 
 COMMANDS_MODULE = ""
 
@@ -267,6 +292,8 @@ DOWNLOAD_HANDLERS_BASE = {
 DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024  # 1024m
 DOWNLOAD_WARNSIZE = 32 * 1024 * 1024  # 32m
 
+DOWNLOAD_SLOTS = {}
+
 DOWNLOAD_TIMEOUT = 180  # 3mins
 
 DOWNLOAD_TLS_MAX_VERSION = None
@@ -304,6 +331,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
 DOWNLOADER_STATS = True
 
 DUPEFILTER_CLASS = "scrapy.dupefilters.RFPDupeFilter"
+DUPEFILTER_DEBUG = False
 
 EDITOR = "vi"
 if sys.platform == "win32":
@@ -354,8 +382,10 @@ FEED_STORAGE_FTP_ACTIVE = False
 FEED_STORAGE_GCS_ACL = ""
 FEED_STORAGE_S3_ACL = ""
 FEED_TEMPDIR = None
+FEED_URI = None
 FEED_URI_PARAMS = None  # a function to extend uri arguments
 
+FILES_STORE = None
 FILES_STORE_GCS_ACL = ""
 FILES_STORE_S3_ACL = "private"
 
@@ -380,11 +410,15 @@ HTTPCACHE_IGNORE_SCHEMES = ["file"]
 HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
 HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 
+HTTPERROR_ALLOW_ALL = False
+HTTPERROR_ALLOWED_CODES = []
+
 HTTPPROXY_ENABLED = True
 HTTPPROXY_AUTH_ENCODING = "latin-1"
 
 HTTPX_HTTP2_ENABLED = False
 
+IMAGES_STORE = None
 IMAGES_STORE_GCS_ACL = ""
 IMAGES_STORE_S3_ACL = "private"
 
@@ -425,6 +459,8 @@ MAIL_HOST = "localhost"
 MAIL_PORT = 25
 MAIL_USER = None
 MAIL_PASS = None
+MAIL_SSL = False
+MAIL_TLS = False
 
 MEMDEBUG_ENABLED = False  # enable memory debugging
 MEMDEBUG_NOTIFY = []  # send memory debugging report by mail at engine shutdown
@@ -455,6 +491,7 @@ REDIRECT_PRIORITY_ADJUST = +2
 
 REFERER_ENABLED = True
 REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
+REFERRER_POLICIES = {}
 
 REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
 

scrapy/spiders/crawl.py

@@ -12,6 +12,7 @@ import warnings
 from collections.abc import AsyncIterator, Awaitable, Callable
 from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.link import Link
 from scrapy.linkextractors import LinkExtractor
@@ -220,4 +221,11 @@ class CrawlSpider(Spider):
     def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
         spider = super().from_crawler(crawler, *args, **kwargs)
         spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS")
+        if not spider._follow_links:
+            warnings.warn(
+                "The CRAWLSPIDER_FOLLOW_LINKS setting is deprecated."
+                " You can set follow=False in your rules to achieve the same effect.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
         return spider

tests/test_spider_crawl.py

@@ -4,6 +4,7 @@ import re
 import warnings
 from logging import ERROR
 
+import pytest
 from testfixtures import LogCapture
 from w3lib.url import safe_url_string
 
@@ -232,6 +233,7 @@ class TestCrawlSpider(TestSpider):
             "HtmlResponse",
         ]
 
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_follow_links_attribute_population(self):
         crawler = get_crawler()
         spider = self.spider_class.from_crawler(crawler, "example.com")