From d047409579023f9b6a15b7e9808576f7addfe89e Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 30 Dec 2008 13:34:57 +0000 Subject: [PATCH] rearranged and sorted out default scrapy settings. closes #20 --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40586 --- scrapy/trunk/scrapy/conf/__init__.py | 8 +- scrapy/trunk/scrapy/conf/core_settings.py | 30 --- scrapy/trunk/scrapy/conf/default_settings.py | 179 ++++++++++++++++++ .../contrib/downloadermiddleware/retry.py | 3 +- 4 files changed, 185 insertions(+), 35 deletions(-) delete mode 100644 scrapy/trunk/scrapy/conf/core_settings.py create mode 100644 scrapy/trunk/scrapy/conf/default_settings.py diff --git a/scrapy/trunk/scrapy/conf/__init__.py b/scrapy/trunk/scrapy/conf/__init__.py index 6bb2f4c6d..48149f3bc 100644 --- a/scrapy/trunk/scrapy/conf/__init__.py +++ b/scrapy/trunk/scrapy/conf/__init__.py @@ -11,14 +11,14 @@ class Settings(object): overrides = None settings = None defaults = None - core = None + global_defaults = None def __init__(self): pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") self.overrides = pickle.loads(pickled_settings) if pickled_settings else {} self.settings = self._import(SETTINGS_MODULE) self.defaults = {} - self.core = self._import('scrapy.conf.core_settings') + self.global_defaults = self._import('scrapy.conf.default_settings') def _import(self, modulepath): return __import__(modulepath, {}, {}, ['']) @@ -36,8 +36,8 @@ class Settings(object): if opt_name in self.defaults: return self.defaults[opt_name] - if hasattr(self.core, opt_name): - return getattr(self.core, opt_name) + if hasattr(self.global_defaults, opt_name): + return getattr(self.global_defaults, opt_name) def get(self, name, default=None): return self[name] if self[name] is not None else default diff --git a/scrapy/trunk/scrapy/conf/core_settings.py b/scrapy/trunk/scrapy/conf/core_settings.py deleted file mode 100644 index 84bb44833..000000000 --- a/scrapy/trunk/scrapy/conf/core_settings.py +++ /dev/null @@ -1,30 +0,0 @@ -import scrapy -# Scrapy core settings - -BOT_NAME = 'scrapy' -BOT_VERSION = scrapy.__version__ - -ENGINE_DEBUG = False - -# Download configuration options -USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) -DOWNLOAD_TIMEOUT = 180 # 3mins -CONCURRENT_DOMAINS = 8 # number of domains to scrape in parallel -REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain -CACHE2_EXPIRATION_SECS = 48 * 60 * 60 # seconds while cached response is still valid (a negative value means "never expires") - -LOG_ENABLED = True # -LOGLEVEL = 'DEBUG' # default loglevel -LOGFILE = None # None means sys.stderr by default -LOG_STDOUT = False # - -DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem' -SCHEDULER = 'scrapy.core.scheduler.Scheduler' -MEMORYSTORE = 'scrapy.core.scheduler.MemoryStore' -PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer' - -EXTENSIONS = [] - -# contrib.middleware.retry.RetryMiddleware default settings -RETRY_TIMES = 2 # initial response + 2 retries = 3 requests -RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200'] diff --git a/scrapy/trunk/scrapy/conf/default_settings.py b/scrapy/trunk/scrapy/conf/default_settings.py new file mode 100644 index 000000000..87de94fde --- /dev/null +++ b/scrapy/trunk/scrapy/conf/default_settings.py @@ -0,0 +1,179 @@ +""" +This module contains the default values for all settings used by Scrapy. + +For more information about these settings you can read the settings +documentation in docs/reference/settings.rst + +Scrapy developers, if you add a setting here remember to: + +* add it in alphabetical order +* group similar settings without leaving blank lines +* add its documentation to the available settings documentation + (docs/reference/settings.rst) + +""" + +from os.path import join, abspath, dirname + +ADAPTORS_DEBUG = False + +BOT_NAME = 'scrapybot' +BOT_VERSION = '1.0' + +CACHE2_DIR = '' +CACHE2_IGNORE_MISSING = False +CACHE2_SECTORIZE = True +CACHE2_EXPIRATION_SECS = 0 + +CLOSEDOMAIN_TIMEOUT = 0 +CLOSEDOMAIN_NOTIFY = ['pablo@insophia.com'] + +CLUSTER_LOGDIR = '' + +CLUSTER_MASTER_PORT = 8790 +CLUSTER_MASTER_ENABLED = 0 +CLUSTER_MASTER_POLL_INTERVAL = 60 +CLUSTER_MASTER_NODES = {} +CLUSTER_MASTER_CACHEFILE = "" + +CLUSTER_WORKER_ENABLED = 0 +CLUSTER_WORKER_MAXPROC = 4 +CLUSTER_WORKER_PORT = 8789 + +COMMANDS_MODULE = '' +COMMANDS_SETTINGS_MODULE = '' + +CONCURRENT_DOMAINS = 8 # number of domains to scrape in parallel + +DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem' + +DEPTH_LIMIT = 0 +DEPTH_STATS = True + +DOWNLOAD_TIMEOUT = 180 # 3mins + +DOWNLOADER_MIDDLEWARES = [ + # Engine side + 'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware', + 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware', + 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware', + 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware', + 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware', + 'scrapy.contrib.downloadermiddleware.common.CommonMiddleware', + 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware', + 'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware', + 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug', + 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats', + 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware', + # Downloader side +] + +DOWNLOADER_STATS = True + +ENABLED_SPIDERS_FILE = '' + +ENGINE_DEBUG = False + +EXTENSIONS = [ + 'scrapy.stats.corestats.CoreStats', + 'scrapy.xpath.extension.ResponseLibxml2', + 'scrapy.management.web.WebConsole', + 'scrapy.management.telnet.TelnetConsole', + 'scrapy.contrib.webconsole.schedstats.SchedulerStats', + 'scrapy.contrib.webconsole.livestats.LiveStats', + 'scrapy.contrib.webconsole.spiderctl.Spiderctl', + 'scrapy.contrib.webconsole.enginestatus.EngineStatus', + 'scrapy.contrib.webconsole.stats.StatsDump', + 'scrapy.contrib.webconsole.spiderstats.SpiderStats', + 'scrapy.contrib.spider.reloader.SpiderReloader', + 'scrapy.contrib.memusage.MemoryUsage', + 'scrapy.contrib.memdebug.MemoryDebugger', + 'scrapy.contrib.pbcluster.ClusterWorker', + 'scrapy.contrib.pbcluster.ClusterMasterWeb', + 'scrapy.contrib.pbcluster.ClusterCrawler', + 'scrapy.contrib.closedomain.CloseDomain', + 'scrapy.contrib.debug.StackTraceDebug', + 'scrapy.contrib.response.soup.ResponseSoup', +] + +GROUPSETTINGS_ENABLED = False +GROUPSETTINGS_MODULE = '' + +# Item pipelines are typically set in specific commands settings +ITEM_PIPELINES = [] + +LOG_ENABLED = True +LOG_STDOUT = False +LOGLEVEL = 'DEBUG' +LOGFILE = None + +MAIL_HOST = 'localhost' +MAIL_FROM = 'scrapy@localhost' + +MEMDEBUG_ENABLED = False # enable memory debugging +MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown + +MEMORYSTORE = 'scrapy.core.scheduler.MemoryStore' + +MEMUSAGE_ENABLED = 1 +MEMUSAGE_LIMIT_MB = 0 +MEMUSAGE_NOTIFY_MAIL = [] +MEMUSAGE_REPORT = False +MEMUSAGE_WARNING_MB = 0 + +MYSQL_CONNECTION_SETTINGS = {} + +NEWSPIDER_MODULE = '' + +PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer' + +REQUESTS_QUEUE_SIZE = 0 +REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain + +# contrib.middleware.retry.RetryMiddleware default settings +RETRY_TIMES = 2 # initial response + 2 retries = 3 requests +RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200'] + +SCHEDULER = 'scrapy.core.scheduler.Scheduler' +SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO + +SPIDER_MODULES = [] + +SPIDERPROFILER_ENABLED = False + +SPIDER_MIDDLEWARES = [ + # Engine side + 'scrapy.contrib.itemsampler.ItemSamplerMiddleware', + 'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware', + 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware', + 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware', + 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware', + 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware', + 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware', + # Spider side +] + +STATS_ENABLED = True +STATS_CLEANUP = False +STATS_DEBUG = False + +# deprecated settings - the stats web service should be moved to the web console +STATS_WSPORT = 8089 +STATS_WSTIMEOUT = 15 + +TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) + +URLLENGTH_LIMIT = 2083 + +USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) + +TELNETCONSOLE_ENABLED = 1 +#TELNETCONSOLE_PORT = 2020 # if not set uses a dynamic port + +WEBCONSOLE_ENABLED = True +WEBCONSOLE_PORT = None +WEBCONSOLE_LOGFILE = None + +# this setting is used by the cluster master to pass additional settings to +# workers at connection time +GLOBAL_CLUSTER_SETTINGS = [] diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py index e7ed17ea5..803c1eb9e 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py @@ -11,7 +11,8 @@ once the spider has finished crawling all regular (non failed) pages. Once there is no more failed pages to retry this middleware sends a signal (retry_complete), so other extensions could connect to that signal. -Default values are located in scrapy.conf.core_settings +Default values are located in scrapy.conf.default_settings, like any other +setting About HTTP errors to consider: