From 05a88152affd1d153c7accb37370919d1374faa5 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sat, 14 Mar 2015 16:12:37 -0300 Subject: [PATCH 1/4] adding more settings to project template --- .../templates/project/module/settings.py.tmpl | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index e9f1ba0e4..572519412 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -2,16 +2,67 @@ # Scrapy settings for $project_name project # -# For simplicity, this file contains only the most important settings by -# default. All the other settings are documented here: +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: # # http://doc.scrapy.org/en/latest/topics/settings.html -# +# http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html +# http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html BOT_NAME = '$project_name' SPIDER_MODULES = ['$project_name.spiders'] NEWSPIDER_MODULE = '$project_name.spiders' + # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = '$project_name (+http://www.yourdomain.com)' + + +# Configure a delay for requests for the same website +# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay +#DOWNLOAD_DELAY=3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN=16 +#CONCURRENT_REQUESTS_PER_IP=16 + +# Disable cookies (enabled by default) +# COOKIES_ENABLED=False + +# Disable Telnet Console (enabled by default) +# TELNETCONSOLE_ENABLED=False + +# Override the default request headers: +# DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +# } + +# Enable or disable spider middlewares +# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html +# SPIDER_MIDDLEWARES = { +# '$project_name.middlewares.MyCustomSpiderMiddleware': 543, +# } + +# Enable or disable downloader middlewares +# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html +# DOWNLOADER_MIDDLEWARES = { +# '$project_name.middlewares.MyCustomDownloaderMiddleware': 543, +# } + +# Enable and configure the AutoThrottle extension (disabled by default) +# See http://doc.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED=True +# The initial download delay (default: 5) +#AUTOTHROTTLE_START_DELAY=3 +# The maximum download delay to be set in case of high latencies (default: 60) +#AUTOTHROTTLE_MAX_DELAY=90 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG=True + +# Enable and configure HTTP caching (disabled by default) +# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED=True +#HTTPCACHE_EXPIRATION_SECS=3600 +#HTTPCACHE_DIR='httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES=[404] From aa56dd30ea31d6ac1785e4fb383375250b2010c9 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Mon, 16 Mar 2015 17:36:46 -0300 Subject: [PATCH 2/4] updating new settings to template, as pointed in PR review --- .../templates/project/module/settings.py.tmpl | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 572519412..f4832dd27 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -18,9 +18,12 @@ NEWSPIDER_MODULE = '$project_name.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = '$project_name (+http://www.yourdomain.com)' +# Configure maximum concurrent requests performed by Scrapy (default: 16) +# CONCURRENT_REQUESTS=32 -# Configure a delay for requests for the same website +# Configure a delay for requests for the same website (default: 0) # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs #DOWNLOAD_DELAY=3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN=16 @@ -50,8 +53,21 @@ NEWSPIDER_MODULE = '$project_name.spiders' # '$project_name.middlewares.MyCustomDownloaderMiddleware': 543, # } +# Enable or disable extensions +# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html +# EXTENSIONS = { +# 'scrapy.telnet.TelnetConsole': None, +# } + +# Configure item pipelines +# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html +# ITEM_PIPELINES = { +# '$project_name.pipelines.SomePipeline': 300, +# } + # Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html +# NOTE: AutoThrottle will honour the standard settings for concurrency and delay #AUTOTHROTTLE_ENABLED=True # The initial download delay (default: 5) #AUTOTHROTTLE_START_DELAY=3 From fd67fe273aa68acbc290d2f067230516bcebcade Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 17 Mar 2015 09:34:54 -0300 Subject: [PATCH 3/4] using default values for settings that are off by default --- scrapy/templates/project/module/settings.py.tmpl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index f4832dd27..302d96b17 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -69,16 +69,17 @@ NEWSPIDER_MODULE = '$project_name.spiders' # See http://doc.scrapy.org/en/latest/topics/autothrottle.html # NOTE: AutoThrottle will honour the standard settings for concurrency and delay #AUTOTHROTTLE_ENABLED=True -# The initial download delay (default: 5) -#AUTOTHROTTLE_START_DELAY=3 -# The maximum download delay to be set in case of high latencies (default: 60) -#AUTOTHROTTLE_MAX_DELAY=90 +# The initial download delay +#AUTOTHROTTLE_START_DELAY=5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY=60 # Enable showing throttling stats for every response received: -#AUTOTHROTTLE_DEBUG=True +#AUTOTHROTTLE_DEBUG=False # Enable and configure HTTP caching (disabled by default) # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED=True -#HTTPCACHE_EXPIRATION_SECS=3600 +#HTTPCACHE_EXPIRATION_SECS=0 #HTTPCACHE_DIR='httpcache' -#HTTPCACHE_IGNORE_HTTP_CODES=[404] +#HTTPCACHE_IGNORE_HTTP_CODES=[] +#HTTPCACHE_STORAGE='scrapy.contrib.httpcache.FilesystemCacheStorage' From d7cb2b9a918dcf4cf4594a05c0c1a3b88fdca413 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 19 Mar 2015 15:41:43 -0300 Subject: [PATCH 4/4] making commented code indentation consistent --- .../templates/project/module/settings.py.tmpl | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 302d96b17..37cbb4d32 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -19,7 +19,7 @@ NEWSPIDER_MODULE = '$project_name.spiders' #USER_AGENT = '$project_name (+http://www.yourdomain.com)' # Configure maximum concurrent requests performed by Scrapy (default: 16) -# CONCURRENT_REQUESTS=32 +#CONCURRENT_REQUESTS=32 # Configure a delay for requests for the same website (default: 0) # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay @@ -30,40 +30,40 @@ NEWSPIDER_MODULE = '$project_name.spiders' #CONCURRENT_REQUESTS_PER_IP=16 # Disable cookies (enabled by default) -# COOKIES_ENABLED=False +#COOKIES_ENABLED=False # Disable Telnet Console (enabled by default) -# TELNETCONSOLE_ENABLED=False +#TELNETCONSOLE_ENABLED=False # Override the default request headers: -# DEFAULT_REQUEST_HEADERS = { -# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', -# 'Accept-Language': 'en', -# } +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} # Enable or disable spider middlewares # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html -# SPIDER_MIDDLEWARES = { -# '$project_name.middlewares.MyCustomSpiderMiddleware': 543, -# } +#SPIDER_MIDDLEWARES = { +# '$project_name.middlewares.MyCustomSpiderMiddleware': 543, +#} # Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html -# DOWNLOADER_MIDDLEWARES = { -# '$project_name.middlewares.MyCustomDownloaderMiddleware': 543, -# } +#DOWNLOADER_MIDDLEWARES = { +# '$project_name.middlewares.MyCustomDownloaderMiddleware': 543, +#} # Enable or disable extensions # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html -# EXTENSIONS = { -# 'scrapy.telnet.TelnetConsole': None, -# } +#EXTENSIONS = { +# 'scrapy.telnet.TelnetConsole': None, +#} # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html -# ITEM_PIPELINES = { -# '$project_name.pipelines.SomePipeline': 300, -# } +#ITEM_PIPELINES = { +# '$project_name.pipelines.SomePipeline': 300, +#} # Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html