diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 037fe87fa..b542c6c05 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -81,9 +81,6 @@ thumbnailing and normalizing images to JPEG/RGB format. Enabling your Media Pipeline ============================ -.. setting:: IMAGES_STORE -.. setting:: FILES_STORE - To enable your media pipeline you must first add it to your project :setting:`ITEM_PIPELINES` setting. @@ -102,6 +99,8 @@ For Files Pipeline, use: .. note:: You can also use both the Files and Images Pipeline at the same time. +.. setting:: IMAGES_STORE +.. setting:: FILES_STORE Then, configure the target storage setting to a valid value that will be used for storing the downloaded images. Otherwise the pipeline will remain disabled, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7d5612026..e8a9400d7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -20,19 +20,26 @@ from typing import Any __all__ = [ "ADDONS", - "AJAXCRAWL_ENABLED", - "AJAXCRAWL_MAXSIZE", "ASYNCIO_EVENT_LOOP", "AUTOTHROTTLE_DEBUG", "AUTOTHROTTLE_ENABLED", "AUTOTHROTTLE_MAX_DELAY", "AUTOTHROTTLE_START_DELAY", "AUTOTHROTTLE_TARGET_CONCURRENCY", + "AWS_ACCESS_KEY_ID", + "AWS_ENDPOINT_URL", + "AWS_REGION_NAME", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_USE_SSL", + "AWS_VERIFY", "BOT_NAME", "CLOSESPIDER_ERRORCOUNT", "CLOSESPIDER_ITEMCOUNT", "CLOSESPIDER_PAGECOUNT", + "CLOSESPIDER_PAGECOUNT_NO_ITEM", "CLOSESPIDER_TIMEOUT", + "CLOSESPIDER_TIMEOUT_NO_ITEM", "COMMANDS_MODULE", "COMPRESSION_ENABLED", "CONCURRENT_ITEMS", @@ -64,11 +71,14 @@ __all__ = [ "DOWNLOAD_HANDLERS", "DOWNLOAD_HANDLERS_BASE", "DOWNLOAD_MAXSIZE", + "DOWNLOAD_SLOTS", "DOWNLOAD_TIMEOUT", "DOWNLOAD_TLS_MAX_VERSION", "DOWNLOAD_TLS_MIN_VERSION", + "DOWNLOAD_VERIFY_CERTIFICATES", "DOWNLOAD_WARNSIZE", "DUPEFILTER_CLASS", + "DUPEFILTER_DEBUG", "EDITOR", "EXTENSIONS", "EXTENSIONS_BASE", @@ -87,7 +97,9 @@ __all__ = [ "FEED_STORAGE_S3_ACL", "FEED_STORE_EMPTY", "FEED_TEMPDIR", + "FEED_URI", "FEED_URI_PARAMS", + "FILES_STORE", "FILES_STORE_GCS_ACL", "FILES_STORE_S3_ACL", "FORCE_CRAWLER_PROCESS", @@ -107,9 +119,12 @@ __all__ = [ "HTTPCACHE_IGNORE_SCHEMES", "HTTPCACHE_POLICY", "HTTPCACHE_STORAGE", + "HTTPERROR_ALLOWED_CODES", + "HTTPERROR_ALLOW_ALL", "HTTPPROXY_AUTH_ENCODING", "HTTPPROXY_ENABLED", "HTTPX_HTTP2_ENABLED", + "IMAGES_STORE", "IMAGES_STORE_GCS_ACL", "IMAGES_STORE_S3_ACL", "ITEM_PIPELINES", @@ -132,6 +147,8 @@ __all__ = [ "MAIL_HOST", "MAIL_PASS", "MAIL_PORT", + "MAIL_SSL", + "MAIL_TLS", "MAIL_USER", "MEMDEBUG_ENABLED", "MEMDEBUG_NOTIFY", @@ -153,6 +170,7 @@ __all__ = [ "REDIRECT_MAX_TIMES", "REDIRECT_PRIORITY_ADJUST", "REFERER_ENABLED", + "REFERRER_POLICIES", "REFERRER_POLICY", "REQUEST_FINGERPRINTER_CLASS", "RETRY_ENABLED", @@ -198,9 +216,6 @@ __all__ = [ ADDONS = {} -AJAXCRAWL_ENABLED = False -AJAXCRAWL_MAXSIZE = 32768 - ASYNCIO_EVENT_LOOP = None AUTOTHROTTLE_ENABLED = False @@ -209,12 +224,22 @@ AUTOTHROTTLE_MAX_DELAY = 60.0 AUTOTHROTTLE_START_DELAY = 5.0 AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +AWS_ACCESS_KEY_ID = None +AWS_SECRET_ACCESS_KEY = None +AWS_ENDPOINT_URL = None +AWS_REGION_NAME = None +AWS_SESSION_TOKEN = None +AWS_USE_SSL = None +AWS_VERIFY = None + BOT_NAME = "scrapybot" CLOSESPIDER_ERRORCOUNT = 0 CLOSESPIDER_ITEMCOUNT = 0 CLOSESPIDER_PAGECOUNT = 0 CLOSESPIDER_TIMEOUT = 0 +CLOSESPIDER_PAGECOUNT_NO_ITEM = 0 +CLOSESPIDER_TIMEOUT_NO_ITEM = 0 COMMANDS_MODULE = "" @@ -267,6 +292,8 @@ DOWNLOAD_HANDLERS_BASE = { DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m +DOWNLOAD_SLOTS = {} + DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOAD_TLS_MAX_VERSION = None @@ -304,6 +331,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { DOWNLOADER_STATS = True DUPEFILTER_CLASS = "scrapy.dupefilters.RFPDupeFilter" +DUPEFILTER_DEBUG = False EDITOR = "vi" if sys.platform == "win32": @@ -354,8 +382,10 @@ FEED_STORAGE_FTP_ACTIVE = False FEED_STORAGE_GCS_ACL = "" FEED_STORAGE_S3_ACL = "" FEED_TEMPDIR = None +FEED_URI = None FEED_URI_PARAMS = None # a function to extend uri arguments +FILES_STORE = None FILES_STORE_GCS_ACL = "" FILES_STORE_S3_ACL = "private" @@ -380,11 +410,15 @@ HTTPCACHE_IGNORE_SCHEMES = ["file"] HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" +HTTPERROR_ALLOW_ALL = False +HTTPERROR_ALLOWED_CODES = [] + HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = "latin-1" HTTPX_HTTP2_ENABLED = False +IMAGES_STORE = None IMAGES_STORE_GCS_ACL = "" IMAGES_STORE_S3_ACL = "private" @@ -425,6 +459,8 @@ MAIL_HOST = "localhost" MAIL_PORT = 25 MAIL_USER = None MAIL_PASS = None +MAIL_SSL = False +MAIL_TLS = False MEMDEBUG_ENABLED = False # enable memory debugging MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown @@ -455,6 +491,7 @@ REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" +REFERRER_POLICIES = {} REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index e14279b64..d2da31f35 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -12,6 +12,7 @@ import warnings from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.link import Link from scrapy.linkextractors import LinkExtractor @@ -220,4 +221,11 @@ class CrawlSpider(Spider): def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS") + if not spider._follow_links: + warnings.warn( + "The CRAWLSPIDER_FOLLOW_LINKS setting is deprecated." + " You can set follow=False in your rules to achieve the same effect.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) return spider diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index 05d77912f..c97934b74 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -4,6 +4,7 @@ import re import warnings from logging import ERROR +import pytest from testfixtures import LogCapture from w3lib.url import safe_url_string @@ -232,6 +233,7 @@ class TestCrawlSpider(TestSpider): "HtmlResponse", ] + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_follow_links_attribute_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com")