mirror of https://github.com/scrapy/scrapy.git
rearranged and sorted out default scrapy settings. closes #20
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40586
This commit is contained in:
Pablo Hoffman
parent
1f247b1efc
commit
d047409579
|
|
@ -11,14 +11,14 @@ class Settings(object):
|
|||
overrides = None
|
||||
settings = None
|
||||
defaults = None
|
||||
core = None
|
||||
global_defaults = None
|
||||
|
||||
def __init__(self):
|
||||
pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
|
||||
self.overrides = pickle.loads(pickled_settings) if pickled_settings else {}
|
||||
self.settings = self._import(SETTINGS_MODULE)
|
||||
self.defaults = {}
|
||||
self.core = self._import('scrapy.conf.core_settings')
|
||||
self.global_defaults = self._import('scrapy.conf.default_settings')
|
||||
|
||||
def _import(self, modulepath):
|
||||
return __import__(modulepath, {}, {}, [''])
|
||||
|
|
@ -36,8 +36,8 @@ class Settings(object):
|
|||
if opt_name in self.defaults:
|
||||
return self.defaults[opt_name]
|
||||
|
||||
if hasattr(self.core, opt_name):
|
||||
return getattr(self.core, opt_name)
|
||||
if hasattr(self.global_defaults, opt_name):
|
||||
return getattr(self.global_defaults, opt_name)
|
||||
|
||||
def get(self, name, default=None):
|
||||
return self[name] if self[name] is not None else default
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
import scrapy
|
||||
# Scrapy core settings
|
||||
|
||||
BOT_NAME = 'scrapy'
|
||||
BOT_VERSION = scrapy.__version__
|
||||
|
||||
ENGINE_DEBUG = False
|
||||
|
||||
# Download configuration options
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
DOWNLOAD_TIMEOUT = 180 # 3mins
|
||||
CONCURRENT_DOMAINS = 8 # number of domains to scrape in parallel
|
||||
REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain
|
||||
CACHE2_EXPIRATION_SECS = 48 * 60 * 60 # seconds while cached response is still valid (a negative value means "never expires")
|
||||
|
||||
LOG_ENABLED = True #
|
||||
LOGLEVEL = 'DEBUG' # default loglevel
|
||||
LOGFILE = None # None means sys.stderr by default
|
||||
LOG_STDOUT = False #
|
||||
|
||||
DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem'
|
||||
SCHEDULER = 'scrapy.core.scheduler.Scheduler'
|
||||
MEMORYSTORE = 'scrapy.core.scheduler.MemoryStore'
|
||||
PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer'
|
||||
|
||||
EXTENSIONS = []
|
||||
|
||||
# contrib.middleware.retry.RetryMiddleware default settings
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200']
|
||||
|
|
@ -0,0 +1,179 @@
|
|||
"""
|
||||
This module contains the default values for all settings used by Scrapy.
|
||||
|
||||
For more information about these settings you can read the settings
|
||||
documentation in docs/reference/settings.rst
|
||||
|
||||
Scrapy developers, if you add a setting here remember to:
|
||||
|
||||
* add it in alphabetical order
|
||||
* group similar settings without leaving blank lines
|
||||
* add its documentation to the available settings documentation
|
||||
(docs/reference/settings.rst)
|
||||
|
||||
"""
|
||||
|
||||
from os.path import join, abspath, dirname
|
||||
|
||||
ADAPTORS_DEBUG = False
|
||||
|
||||
BOT_NAME = 'scrapybot'
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
CACHE2_DIR = ''
|
||||
CACHE2_IGNORE_MISSING = False
|
||||
CACHE2_SECTORIZE = True
|
||||
CACHE2_EXPIRATION_SECS = 0
|
||||
|
||||
CLOSEDOMAIN_TIMEOUT = 0
|
||||
CLOSEDOMAIN_NOTIFY = ['pablo@insophia.com']
|
||||
|
||||
CLUSTER_LOGDIR = ''
|
||||
|
||||
CLUSTER_MASTER_PORT = 8790
|
||||
CLUSTER_MASTER_ENABLED = 0
|
||||
CLUSTER_MASTER_POLL_INTERVAL = 60
|
||||
CLUSTER_MASTER_NODES = {}
|
||||
CLUSTER_MASTER_CACHEFILE = ""
|
||||
|
||||
CLUSTER_WORKER_ENABLED = 0
|
||||
CLUSTER_WORKER_MAXPROC = 4
|
||||
CLUSTER_WORKER_PORT = 8789
|
||||
|
||||
COMMANDS_MODULE = ''
|
||||
COMMANDS_SETTINGS_MODULE = ''
|
||||
|
||||
CONCURRENT_DOMAINS = 8 # number of domains to scrape in parallel
|
||||
|
||||
DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem'
|
||||
|
||||
DEPTH_LIMIT = 0
|
||||
DEPTH_STATS = True
|
||||
|
||||
DOWNLOAD_TIMEOUT = 180 # 3mins
|
||||
|
||||
DOWNLOADER_MIDDLEWARES = [
|
||||
# Engine side
|
||||
'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.common.CommonMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.debug.CrawlDebug',
|
||||
'scrapy.contrib.downloadermiddleware.stats.DownloaderStats',
|
||||
'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware',
|
||||
# Downloader side
|
||||
]
|
||||
|
||||
DOWNLOADER_STATS = True
|
||||
|
||||
ENABLED_SPIDERS_FILE = ''
|
||||
|
||||
ENGINE_DEBUG = False
|
||||
|
||||
EXTENSIONS = [
|
||||
'scrapy.stats.corestats.CoreStats',
|
||||
'scrapy.xpath.extension.ResponseLibxml2',
|
||||
'scrapy.management.web.WebConsole',
|
||||
'scrapy.management.telnet.TelnetConsole',
|
||||
'scrapy.contrib.webconsole.schedstats.SchedulerStats',
|
||||
'scrapy.contrib.webconsole.livestats.LiveStats',
|
||||
'scrapy.contrib.webconsole.spiderctl.Spiderctl',
|
||||
'scrapy.contrib.webconsole.enginestatus.EngineStatus',
|
||||
'scrapy.contrib.webconsole.stats.StatsDump',
|
||||
'scrapy.contrib.webconsole.spiderstats.SpiderStats',
|
||||
'scrapy.contrib.spider.reloader.SpiderReloader',
|
||||
'scrapy.contrib.memusage.MemoryUsage',
|
||||
'scrapy.contrib.memdebug.MemoryDebugger',
|
||||
'scrapy.contrib.pbcluster.ClusterWorker',
|
||||
'scrapy.contrib.pbcluster.ClusterMasterWeb',
|
||||
'scrapy.contrib.pbcluster.ClusterCrawler',
|
||||
'scrapy.contrib.closedomain.CloseDomain',
|
||||
'scrapy.contrib.debug.StackTraceDebug',
|
||||
'scrapy.contrib.response.soup.ResponseSoup',
|
||||
]
|
||||
|
||||
GROUPSETTINGS_ENABLED = False
|
||||
GROUPSETTINGS_MODULE = ''
|
||||
|
||||
# Item pipelines are typically set in specific commands settings
|
||||
ITEM_PIPELINES = []
|
||||
|
||||
LOG_ENABLED = True
|
||||
LOG_STDOUT = False
|
||||
LOGLEVEL = 'DEBUG'
|
||||
LOGFILE = None
|
||||
|
||||
MAIL_HOST = 'localhost'
|
||||
MAIL_FROM = 'scrapy@localhost'
|
||||
|
||||
MEMDEBUG_ENABLED = False # enable memory debugging
|
||||
MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown
|
||||
|
||||
MEMORYSTORE = 'scrapy.core.scheduler.MemoryStore'
|
||||
|
||||
MEMUSAGE_ENABLED = 1
|
||||
MEMUSAGE_LIMIT_MB = 0
|
||||
MEMUSAGE_NOTIFY_MAIL = []
|
||||
MEMUSAGE_REPORT = False
|
||||
MEMUSAGE_WARNING_MB = 0
|
||||
|
||||
MYSQL_CONNECTION_SETTINGS = {}
|
||||
|
||||
NEWSPIDER_MODULE = ''
|
||||
|
||||
PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer'
|
||||
|
||||
REQUESTS_QUEUE_SIZE = 0
|
||||
REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain
|
||||
|
||||
# contrib.middleware.retry.RetryMiddleware default settings
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200']
|
||||
|
||||
SCHEDULER = 'scrapy.core.scheduler.Scheduler'
|
||||
SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO
|
||||
|
||||
SPIDER_MODULES = []
|
||||
|
||||
SPIDERPROFILER_ENABLED = False
|
||||
|
||||
SPIDER_MIDDLEWARES = [
|
||||
# Engine side
|
||||
'scrapy.contrib.itemsampler.ItemSamplerMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.referer.RefererMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.depth.DepthMiddleware',
|
||||
# Spider side
|
||||
]
|
||||
|
||||
STATS_ENABLED = True
|
||||
STATS_CLEANUP = False
|
||||
STATS_DEBUG = False
|
||||
|
||||
# deprecated settings - the stats web service should be moved to the web console
|
||||
STATS_WSPORT = 8089
|
||||
STATS_WSTIMEOUT = 15
|
||||
|
||||
TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
|
||||
|
||||
URLLENGTH_LIMIT = 2083
|
||||
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
|
||||
TELNETCONSOLE_ENABLED = 1
|
||||
#TELNETCONSOLE_PORT = 2020 # if not set uses a dynamic port
|
||||
|
||||
WEBCONSOLE_ENABLED = True
|
||||
WEBCONSOLE_PORT = None
|
||||
WEBCONSOLE_LOGFILE = None
|
||||
|
||||
# this setting is used by the cluster master to pass additional settings to
|
||||
# workers at connection time
|
||||
GLOBAL_CLUSTER_SETTINGS = []
|
||||
|
|
@ -11,7 +11,8 @@ once the spider has finished crawling all regular (non failed) pages. Once
|
|||
there is no more failed pages to retry this middleware sends a signal
|
||||
(retry_complete), so other extensions could connect to that signal.
|
||||
|
||||
Default values are located in scrapy.conf.core_settings
|
||||
Default values are located in scrapy.conf.default_settings, like any other
|
||||
setting
|
||||
|
||||
About HTTP errors to consider:
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue