adding more settings to project template

scrapy/templates/project/module/settings.py.tmpl

@@ -2,16 +2,67 @@
 
 # Scrapy settings for $project_name project
 #
-# For simplicity, this file contains only the most important settings by
-# default. All the other settings are documented here:
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
 #
 #     http://doc.scrapy.org/en/latest/topics/settings.html
-#
+#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
+#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
 
 BOT_NAME = '$project_name'
 
 SPIDER_MODULES = ['$project_name.spiders']
 NEWSPIDER_MODULE = '$project_name.spiders'
 
+
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 #USER_AGENT = '$project_name (+http://www.yourdomain.com)'
+
+
+# Configure a delay for requests for the same website
+# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
+#DOWNLOAD_DELAY=3
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN=16
+#CONCURRENT_REQUESTS_PER_IP=16
+
+# Disable cookies (enabled by default)
+# COOKIES_ENABLED=False
+
+# Disable Telnet Console (enabled by default)
+# TELNETCONSOLE_ENABLED=False
+
+# Override the default request headers:
+# DEFAULT_REQUEST_HEADERS = {
+#    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#    'Accept-Language': 'en',
+# }
+
+# Enable or disable spider middlewares
+# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
+# SPIDER_MIDDLEWARES = {
+#     '$project_name.middlewares.MyCustomSpiderMiddleware': 543,
+# }
+
+# Enable or disable downloader middlewares
+# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
+# DOWNLOADER_MIDDLEWARES = {
+#     '$project_name.middlewares.MyCustomDownloaderMiddleware': 543,
+# }
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See http://doc.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED=True
+# The initial download delay (default: 5)
+#AUTOTHROTTLE_START_DELAY=3
+# The maximum download delay to be set in case of high latencies (default: 60)
+#AUTOTHROTTLE_MAX_DELAY=90
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG=True
+
+# Enable and configure HTTP caching (disabled by default)
+# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED=True
+#HTTPCACHE_EXPIRATION_SECS=3600
+#HTTPCACHE_DIR='httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES=[404]