mirror of https://github.com/scrapy/scrapy.git
fixing settings order (#6849)
* fixing issue #6838 * Reorder some more settings. * Clarify the header. --------- Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
This commit is contained in:
Mehraz Hossain Rumman
GitHub
parent
8fb8d2c6b8
commit
c6698b9fe8
|
|
@ -1,16 +1,16 @@
|
|||
"""
|
||||
This module contains the default values for all settings used by Scrapy.
|
||||
"""This module contains the default values for all settings used by Scrapy.
|
||||
|
||||
For more information about these settings you can read the settings
|
||||
documentation in docs/topics/settings.rst
|
||||
|
||||
Scrapy developers, if you add a setting here remember to:
|
||||
|
||||
* add it in alphabetical order
|
||||
* add it in alphabetical order, with the exception that enabling flags and
|
||||
other high-level settings for a group should come first in their group
|
||||
and pairs like host/port and user/password should be in the usual order
|
||||
* group similar settings without leaving blank lines
|
||||
* add its documentation to the available settings documentation
|
||||
(docs/topics/settings.rst)
|
||||
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
|
@ -31,10 +31,10 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
|||
|
||||
BOT_NAME = "scrapybot"
|
||||
|
||||
CLOSESPIDER_TIMEOUT = 0
|
||||
CLOSESPIDER_PAGECOUNT = 0
|
||||
CLOSESPIDER_ITEMCOUNT = 0
|
||||
CLOSESPIDER_ERRORCOUNT = 0
|
||||
CLOSESPIDER_ITEMCOUNT = 0
|
||||
CLOSESPIDER_PAGECOUNT = 0
|
||||
CLOSESPIDER_TIMEOUT = 0
|
||||
|
||||
COMMANDS_MODULE = ""
|
||||
|
||||
|
|
@ -59,8 +59,8 @@ DEFAULT_REQUEST_HEADERS = {
|
|||
}
|
||||
|
||||
DEPTH_LIMIT = 0
|
||||
DEPTH_STATS_VERBOSE = False
|
||||
DEPTH_PRIORITY = 0
|
||||
DEPTH_STATS_VERBOSE = False
|
||||
|
||||
DNSCACHE_ENABLED = True
|
||||
DNSCACHE_SIZE = 10000
|
||||
|
|
@ -69,6 +69,8 @@ DNS_TIMEOUT = 60
|
|||
|
||||
DOWNLOAD_DELAY = 0
|
||||
|
||||
DOWNLOAD_FAIL_ON_DATALOSS = True
|
||||
|
||||
DOWNLOAD_HANDLERS = {}
|
||||
DOWNLOAD_HANDLERS_BASE = {
|
||||
"data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler",
|
||||
|
|
@ -79,18 +81,13 @@ DOWNLOAD_HANDLERS_BASE = {
|
|||
"ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler",
|
||||
}
|
||||
|
||||
DOWNLOAD_TIMEOUT = 180 # 3mins
|
||||
|
||||
DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m
|
||||
DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m
|
||||
|
||||
DOWNLOAD_FAIL_ON_DATALOSS = True
|
||||
DOWNLOAD_TIMEOUT = 180 # 3mins
|
||||
|
||||
DOWNLOADER = "scrapy.core.downloader.Downloader"
|
||||
|
||||
DOWNLOADER_HTTPCLIENTFACTORY = (
|
||||
"scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
|
||||
)
|
||||
DOWNLOADER_CLIENTCONTEXTFACTORY = (
|
||||
"scrapy.core.downloader.contextfactory.ScrapyClientContextFactory"
|
||||
)
|
||||
|
|
@ -99,8 +96,11 @@ DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT"
|
|||
DOWNLOADER_CLIENT_TLS_METHOD = "TLS"
|
||||
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
|
||||
|
||||
DOWNLOADER_MIDDLEWARES = {}
|
||||
DOWNLOADER_HTTPCLIENTFACTORY = (
|
||||
"scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
|
||||
)
|
||||
|
||||
DOWNLOADER_MIDDLEWARES = {}
|
||||
DOWNLOADER_MIDDLEWARES_BASE = {
|
||||
# Engine side
|
||||
"scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50,
|
||||
|
|
@ -130,7 +130,6 @@ if sys.platform == "win32":
|
|||
EDITOR = "%s -m idlelib.idle"
|
||||
|
||||
EXTENSIONS = {}
|
||||
|
||||
EXTENSIONS_BASE = {
|
||||
"scrapy.extensions.corestats.CoreStats": 0,
|
||||
"scrapy.extensions.telnet.TelnetConsole": 0,
|
||||
|
|
@ -143,22 +142,11 @@ EXTENSIONS_BASE = {
|
|||
"scrapy.extensions.throttle.AutoThrottle": 0,
|
||||
}
|
||||
|
||||
FEED_TEMPDIR = None
|
||||
FEEDS = {}
|
||||
FEED_URI_PARAMS = None # a function to extend uri arguments
|
||||
FEED_STORE_EMPTY = True
|
||||
FEED_EXPORT_BATCH_ITEM_COUNT = 0
|
||||
FEED_EXPORT_ENCODING = None
|
||||
FEED_EXPORT_FIELDS = None
|
||||
FEED_STORAGES = {}
|
||||
FEED_STORAGES_BASE = {
|
||||
"": "scrapy.extensions.feedexport.FileFeedStorage",
|
||||
"file": "scrapy.extensions.feedexport.FileFeedStorage",
|
||||
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
|
||||
"gs": "scrapy.extensions.feedexport.GCSFeedStorage",
|
||||
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
|
||||
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
|
||||
}
|
||||
FEED_EXPORT_BATCH_ITEM_COUNT = 0
|
||||
FEED_EXPORT_INDENT = 0
|
||||
FEED_EXPORTERS = {}
|
||||
FEED_EXPORTERS_BASE = {
|
||||
"json": "scrapy.exporters.JsonItemExporter",
|
||||
|
|
@ -170,59 +158,69 @@ FEED_EXPORTERS_BASE = {
|
|||
"marshal": "scrapy.exporters.MarshalItemExporter",
|
||||
"pickle": "scrapy.exporters.PickleItemExporter",
|
||||
}
|
||||
FEED_EXPORT_INDENT = 0
|
||||
|
||||
FEED_STORE_EMPTY = True
|
||||
FEED_STORAGES = {}
|
||||
FEED_STORAGES_BASE = {
|
||||
"": "scrapy.extensions.feedexport.FileFeedStorage",
|
||||
"file": "scrapy.extensions.feedexport.FileFeedStorage",
|
||||
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
|
||||
"gs": "scrapy.extensions.feedexport.GCSFeedStorage",
|
||||
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
|
||||
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
|
||||
}
|
||||
FEED_STORAGE_FTP_ACTIVE = False
|
||||
FEED_STORAGE_GCS_ACL = ""
|
||||
FEED_STORAGE_S3_ACL = ""
|
||||
FEED_TEMPDIR = None
|
||||
FEED_URI_PARAMS = None # a function to extend uri arguments
|
||||
|
||||
FILES_STORE_S3_ACL = "private"
|
||||
FILES_STORE_GCS_ACL = ""
|
||||
FILES_STORE_S3_ACL = "private"
|
||||
|
||||
FORCE_CRAWLER_PROCESS = False
|
||||
|
||||
FTP_PASSIVE_MODE = True
|
||||
FTP_USER = "anonymous"
|
||||
FTP_PASSWORD = "guest" # noqa: S105
|
||||
FTP_PASSIVE_MODE = True
|
||||
|
||||
GCS_PROJECT_ID = None
|
||||
|
||||
HTTPCACHE_ENABLED = False
|
||||
HTTPCACHE_DIR = "httpcache"
|
||||
HTTPCACHE_IGNORE_MISSING = False
|
||||
HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
|
||||
HTTPCACHE_EXPIRATION_SECS = 0
|
||||
HTTPCACHE_ALWAYS_STORE = False
|
||||
HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
HTTPCACHE_IGNORE_SCHEMES = ["file"]
|
||||
HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
|
||||
HTTPCACHE_DBM_MODULE = "dbm"
|
||||
HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
|
||||
HTTPCACHE_DIR = "httpcache"
|
||||
HTTPCACHE_EXPIRATION_SECS = 0
|
||||
HTTPCACHE_GZIP = False
|
||||
HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
HTTPCACHE_IGNORE_MISSING = False
|
||||
HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
|
||||
HTTPCACHE_IGNORE_SCHEMES = ["file"]
|
||||
HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
|
||||
HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
|
||||
|
||||
HTTPPROXY_ENABLED = True
|
||||
HTTPPROXY_AUTH_ENCODING = "latin-1"
|
||||
|
||||
IMAGES_STORE_S3_ACL = "private"
|
||||
IMAGES_STORE_GCS_ACL = ""
|
||||
|
||||
ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"
|
||||
IMAGES_STORE_S3_ACL = "private"
|
||||
|
||||
ITEM_PIPELINES = {}
|
||||
ITEM_PIPELINES_BASE = {}
|
||||
|
||||
ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"
|
||||
|
||||
JOBDIR = None
|
||||
|
||||
LOG_ENABLED = True
|
||||
LOG_ENCODING = "utf-8"
|
||||
LOG_FORMATTER = "scrapy.logformatter.LogFormatter"
|
||||
LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
|
||||
LOG_DATEFORMAT = "%Y-%m-%d %H:%M:%S"
|
||||
LOG_STDOUT = False
|
||||
LOG_LEVEL = "DEBUG"
|
||||
LOG_ENCODING = "utf-8"
|
||||
LOG_FILE = None
|
||||
LOG_FILE_APPEND = True
|
||||
LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
|
||||
LOG_FORMATTER = "scrapy.logformatter.LogFormatter"
|
||||
LOG_LEVEL = "DEBUG"
|
||||
LOG_SHORT_NAMES = False
|
||||
LOG_STDOUT = False
|
||||
LOG_VERSIONS = [
|
||||
"lxml",
|
||||
"libxml2",
|
||||
|
|
@ -236,21 +234,19 @@ LOG_VERSIONS = [
|
|||
"Platform",
|
||||
]
|
||||
|
||||
SCHEDULER_DEBUG = False
|
||||
|
||||
LOGSTATS_INTERVAL = 60.0
|
||||
|
||||
MAIL_FROM = "scrapy@localhost"
|
||||
MAIL_HOST = "localhost"
|
||||
MAIL_PORT = 25
|
||||
MAIL_FROM = "scrapy@localhost"
|
||||
MAIL_PASS = None
|
||||
MAIL_USER = None
|
||||
MAIL_PASS = None
|
||||
|
||||
MEMDEBUG_ENABLED = False # enable memory debugging
|
||||
MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown
|
||||
|
||||
MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
|
||||
MEMUSAGE_ENABLED = True
|
||||
MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
|
||||
MEMUSAGE_LIMIT_MB = 0
|
||||
MEMUSAGE_NOTIFY_MAIL = []
|
||||
MEMUSAGE_WARNING_MB = 0
|
||||
|
|
@ -280,9 +276,6 @@ REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
|
|||
REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL"
|
||||
|
||||
RETRY_ENABLED = True
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
|
||||
RETRY_PRIORITY_ADJUST = -1
|
||||
RETRY_EXCEPTIONS = [
|
||||
"twisted.internet.defer.TimeoutError",
|
||||
"twisted.internet.error.TimeoutError",
|
||||
|
|
@ -298,12 +291,16 @@ RETRY_EXCEPTIONS = [
|
|||
OSError,
|
||||
"scrapy.core.downloader.handlers.http11.TunnelError",
|
||||
]
|
||||
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
|
||||
RETRY_PRIORITY_ADJUST = -1
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
|
||||
ROBOTSTXT_OBEY = False
|
||||
ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser"
|
||||
ROBOTSTXT_USER_AGENT = None
|
||||
|
||||
SCHEDULER = "scrapy.core.scheduler.Scheduler"
|
||||
SCHEDULER_DEBUG = False
|
||||
SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue"
|
||||
SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue"
|
||||
SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue"
|
||||
|
|
@ -312,11 +309,19 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"
|
|||
|
||||
SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000
|
||||
|
||||
SPIDER_CONTRACTS = {}
|
||||
SPIDER_CONTRACTS_BASE = {
|
||||
"scrapy.contracts.default.UrlContract": 1,
|
||||
"scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
|
||||
"scrapy.contracts.default.MetadataContract": 1,
|
||||
"scrapy.contracts.default.ReturnsContract": 2,
|
||||
"scrapy.contracts.default.ScrapesContract": 3,
|
||||
}
|
||||
|
||||
SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader"
|
||||
SPIDER_LOADER_WARN_ONLY = False
|
||||
|
||||
SPIDER_MIDDLEWARES = {}
|
||||
|
||||
SPIDER_MIDDLEWARES_BASE = {
|
||||
# Engine side
|
||||
"scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,
|
||||
|
|
@ -334,27 +339,18 @@ STATS_DUMP = True
|
|||
|
||||
STATSMAILER_RCPTS = []
|
||||
|
||||
TELNETCONSOLE_ENABLED = 1
|
||||
TELNETCONSOLE_HOST = "127.0.0.1"
|
||||
TELNETCONSOLE_PORT = [6023, 6073]
|
||||
TELNETCONSOLE_USERNAME = "scrapy"
|
||||
TELNETCONSOLE_PASSWORD = None
|
||||
|
||||
TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())
|
||||
|
||||
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
|
||||
|
||||
URLLENGTH_LIMIT = 2083
|
||||
|
||||
USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)"
|
||||
|
||||
TELNETCONSOLE_ENABLED = 1
|
||||
TELNETCONSOLE_PORT = [6023, 6073]
|
||||
TELNETCONSOLE_HOST = "127.0.0.1"
|
||||
TELNETCONSOLE_USERNAME = "scrapy"
|
||||
TELNETCONSOLE_PASSWORD = None
|
||||
|
||||
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
|
||||
|
||||
SPIDER_CONTRACTS = {}
|
||||
SPIDER_CONTRACTS_BASE = {
|
||||
"scrapy.contracts.default.UrlContract": 1,
|
||||
"scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
|
||||
"scrapy.contracts.default.MetadataContract": 1,
|
||||
"scrapy.contracts.default.ReturnsContract": 2,
|
||||
"scrapy.contracts.default.ScrapesContract": 3,
|
||||
}
|
||||
|
||||
WARN_ON_GENERATOR_RETURN_VALUE = True
|
||||
|
|
|
|||
Loading…
Reference in New Issue