RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

updated default_settings with robots.txt related settings 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40631
This commit is contained in:
Pablo Hoffman 2009-01-03 07:36:35 +00:00
parent e437ff67ae
commit 4a3ba6957b
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scrapy/trunk/scrapy/conf/default_settings.py
View File

@ -54,6 +54,7 @@ DOWNLOAD_TIMEOUT = 180 # 3mins
DOWNLOADER_MIDDLEWARES = [
# Engine side
'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware',
'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware',
'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware',
'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware',
@ -134,6 +135,8 @@ REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200']
ROBOTSTXT_OBEY = True
SCHEDULER = 'scrapy.core.scheduler.Scheduler'
SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO