mirror of https://github.com/scrapy/scrapy.git
Update default_settings, deprecate CRAWLSPIDER_FOLLOW_LINKS. (#7592)
This commit is contained in:
Andrey Rakhmatullin
GitHub
parent
d59f9b644a
commit
beb6c51c17
|
|
@ -81,9 +81,6 @@ thumbnailing and normalizing images to JPEG/RGB format.
|
|||
Enabling your Media Pipeline
|
||||
============================
|
||||
|
||||
.. setting:: IMAGES_STORE
|
||||
.. setting:: FILES_STORE
|
||||
|
||||
To enable your media pipeline you must first add it to your project
|
||||
:setting:`ITEM_PIPELINES` setting.
|
||||
|
||||
|
|
@ -102,6 +99,8 @@ For Files Pipeline, use:
|
|||
.. note::
|
||||
You can also use both the Files and Images Pipeline at the same time.
|
||||
|
||||
.. setting:: IMAGES_STORE
|
||||
.. setting:: FILES_STORE
|
||||
|
||||
Then, configure the target storage setting to a valid value that will be used
|
||||
for storing the downloaded images. Otherwise the pipeline will remain disabled,
|
||||
|
|
|
|||
|
|
@ -20,19 +20,26 @@ from typing import Any
|
|||
|
||||
__all__ = [
|
||||
"ADDONS",
|
||||
"AJAXCRAWL_ENABLED",
|
||||
"AJAXCRAWL_MAXSIZE",
|
||||
"ASYNCIO_EVENT_LOOP",
|
||||
"AUTOTHROTTLE_DEBUG",
|
||||
"AUTOTHROTTLE_ENABLED",
|
||||
"AUTOTHROTTLE_MAX_DELAY",
|
||||
"AUTOTHROTTLE_START_DELAY",
|
||||
"AUTOTHROTTLE_TARGET_CONCURRENCY",
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"AWS_ENDPOINT_URL",
|
||||
"AWS_REGION_NAME",
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"AWS_SESSION_TOKEN",
|
||||
"AWS_USE_SSL",
|
||||
"AWS_VERIFY",
|
||||
"BOT_NAME",
|
||||
"CLOSESPIDER_ERRORCOUNT",
|
||||
"CLOSESPIDER_ITEMCOUNT",
|
||||
"CLOSESPIDER_PAGECOUNT",
|
||||
"CLOSESPIDER_PAGECOUNT_NO_ITEM",
|
||||
"CLOSESPIDER_TIMEOUT",
|
||||
"CLOSESPIDER_TIMEOUT_NO_ITEM",
|
||||
"COMMANDS_MODULE",
|
||||
"COMPRESSION_ENABLED",
|
||||
"CONCURRENT_ITEMS",
|
||||
|
|
@ -64,11 +71,14 @@ __all__ = [
|
|||
"DOWNLOAD_HANDLERS",
|
||||
"DOWNLOAD_HANDLERS_BASE",
|
||||
"DOWNLOAD_MAXSIZE",
|
||||
"DOWNLOAD_SLOTS",
|
||||
"DOWNLOAD_TIMEOUT",
|
||||
"DOWNLOAD_TLS_MAX_VERSION",
|
||||
"DOWNLOAD_TLS_MIN_VERSION",
|
||||
"DOWNLOAD_VERIFY_CERTIFICATES",
|
||||
"DOWNLOAD_WARNSIZE",
|
||||
"DUPEFILTER_CLASS",
|
||||
"DUPEFILTER_DEBUG",
|
||||
"EDITOR",
|
||||
"EXTENSIONS",
|
||||
"EXTENSIONS_BASE",
|
||||
|
|
@ -87,7 +97,9 @@ __all__ = [
|
|||
"FEED_STORAGE_S3_ACL",
|
||||
"FEED_STORE_EMPTY",
|
||||
"FEED_TEMPDIR",
|
||||
"FEED_URI",
|
||||
"FEED_URI_PARAMS",
|
||||
"FILES_STORE",
|
||||
"FILES_STORE_GCS_ACL",
|
||||
"FILES_STORE_S3_ACL",
|
||||
"FORCE_CRAWLER_PROCESS",
|
||||
|
|
@ -107,9 +119,12 @@ __all__ = [
|
|||
"HTTPCACHE_IGNORE_SCHEMES",
|
||||
"HTTPCACHE_POLICY",
|
||||
"HTTPCACHE_STORAGE",
|
||||
"HTTPERROR_ALLOWED_CODES",
|
||||
"HTTPERROR_ALLOW_ALL",
|
||||
"HTTPPROXY_AUTH_ENCODING",
|
||||
"HTTPPROXY_ENABLED",
|
||||
"HTTPX_HTTP2_ENABLED",
|
||||
"IMAGES_STORE",
|
||||
"IMAGES_STORE_GCS_ACL",
|
||||
"IMAGES_STORE_S3_ACL",
|
||||
"ITEM_PIPELINES",
|
||||
|
|
@ -132,6 +147,8 @@ __all__ = [
|
|||
"MAIL_HOST",
|
||||
"MAIL_PASS",
|
||||
"MAIL_PORT",
|
||||
"MAIL_SSL",
|
||||
"MAIL_TLS",
|
||||
"MAIL_USER",
|
||||
"MEMDEBUG_ENABLED",
|
||||
"MEMDEBUG_NOTIFY",
|
||||
|
|
@ -153,6 +170,7 @@ __all__ = [
|
|||
"REDIRECT_MAX_TIMES",
|
||||
"REDIRECT_PRIORITY_ADJUST",
|
||||
"REFERER_ENABLED",
|
||||
"REFERRER_POLICIES",
|
||||
"REFERRER_POLICY",
|
||||
"REQUEST_FINGERPRINTER_CLASS",
|
||||
"RETRY_ENABLED",
|
||||
|
|
@ -198,9 +216,6 @@ __all__ = [
|
|||
|
||||
ADDONS = {}
|
||||
|
||||
AJAXCRAWL_ENABLED = False
|
||||
AJAXCRAWL_MAXSIZE = 32768
|
||||
|
||||
ASYNCIO_EVENT_LOOP = None
|
||||
|
||||
AUTOTHROTTLE_ENABLED = False
|
||||
|
|
@ -209,12 +224,22 @@ AUTOTHROTTLE_MAX_DELAY = 60.0
|
|||
AUTOTHROTTLE_START_DELAY = 5.0
|
||||
AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
||||
|
||||
AWS_ACCESS_KEY_ID = None
|
||||
AWS_SECRET_ACCESS_KEY = None
|
||||
AWS_ENDPOINT_URL = None
|
||||
AWS_REGION_NAME = None
|
||||
AWS_SESSION_TOKEN = None
|
||||
AWS_USE_SSL = None
|
||||
AWS_VERIFY = None
|
||||
|
||||
BOT_NAME = "scrapybot"
|
||||
|
||||
CLOSESPIDER_ERRORCOUNT = 0
|
||||
CLOSESPIDER_ITEMCOUNT = 0
|
||||
CLOSESPIDER_PAGECOUNT = 0
|
||||
CLOSESPIDER_TIMEOUT = 0
|
||||
CLOSESPIDER_PAGECOUNT_NO_ITEM = 0
|
||||
CLOSESPIDER_TIMEOUT_NO_ITEM = 0
|
||||
|
||||
COMMANDS_MODULE = ""
|
||||
|
||||
|
|
@ -267,6 +292,8 @@ DOWNLOAD_HANDLERS_BASE = {
|
|||
DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m
|
||||
DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m
|
||||
|
||||
DOWNLOAD_SLOTS = {}
|
||||
|
||||
DOWNLOAD_TIMEOUT = 180 # 3mins
|
||||
|
||||
DOWNLOAD_TLS_MAX_VERSION = None
|
||||
|
|
@ -304,6 +331,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
|
|||
DOWNLOADER_STATS = True
|
||||
|
||||
DUPEFILTER_CLASS = "scrapy.dupefilters.RFPDupeFilter"
|
||||
DUPEFILTER_DEBUG = False
|
||||
|
||||
EDITOR = "vi"
|
||||
if sys.platform == "win32":
|
||||
|
|
@ -354,8 +382,10 @@ FEED_STORAGE_FTP_ACTIVE = False
|
|||
FEED_STORAGE_GCS_ACL = ""
|
||||
FEED_STORAGE_S3_ACL = ""
|
||||
FEED_TEMPDIR = None
|
||||
FEED_URI = None
|
||||
FEED_URI_PARAMS = None # a function to extend uri arguments
|
||||
|
||||
FILES_STORE = None
|
||||
FILES_STORE_GCS_ACL = ""
|
||||
FILES_STORE_S3_ACL = "private"
|
||||
|
||||
|
|
@ -380,11 +410,15 @@ HTTPCACHE_IGNORE_SCHEMES = ["file"]
|
|||
HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
|
||||
HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
|
||||
|
||||
HTTPERROR_ALLOW_ALL = False
|
||||
HTTPERROR_ALLOWED_CODES = []
|
||||
|
||||
HTTPPROXY_ENABLED = True
|
||||
HTTPPROXY_AUTH_ENCODING = "latin-1"
|
||||
|
||||
HTTPX_HTTP2_ENABLED = False
|
||||
|
||||
IMAGES_STORE = None
|
||||
IMAGES_STORE_GCS_ACL = ""
|
||||
IMAGES_STORE_S3_ACL = "private"
|
||||
|
||||
|
|
@ -425,6 +459,8 @@ MAIL_HOST = "localhost"
|
|||
MAIL_PORT = 25
|
||||
MAIL_USER = None
|
||||
MAIL_PASS = None
|
||||
MAIL_SSL = False
|
||||
MAIL_TLS = False
|
||||
|
||||
MEMDEBUG_ENABLED = False # enable memory debugging
|
||||
MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown
|
||||
|
|
@ -455,6 +491,7 @@ REDIRECT_PRIORITY_ADJUST = +2
|
|||
|
||||
REFERER_ENABLED = True
|
||||
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
|
||||
REFERRER_POLICIES = {}
|
||||
|
||||
REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import warnings
|
|||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import HtmlResponse, Request, Response
|
||||
from scrapy.link import Link
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
|
|
@ -220,4 +221,11 @@ class CrawlSpider(Spider):
|
|||
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
|
||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
||||
spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS")
|
||||
if not spider._follow_links:
|
||||
warnings.warn(
|
||||
"The CRAWLSPIDER_FOLLOW_LINKS setting is deprecated."
|
||||
" You can set follow=False in your rules to achieve the same effect.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return spider
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import re
|
|||
import warnings
|
||||
from logging import ERROR
|
||||
|
||||
import pytest
|
||||
from testfixtures import LogCapture
|
||||
from w3lib.url import safe_url_string
|
||||
|
||||
|
|
@ -232,6 +233,7 @@ class TestCrawlSpider(TestSpider):
|
|||
"HtmlResponse",
|
||||
]
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_follow_links_attribute_population(self):
|
||||
crawler = get_crawler()
|
||||
spider = self.spider_class.from_crawler(crawler, "example.com")
|
||||
|
|
|
|||
Loading…
Reference in New Issue