From 4a3ba6957bc0b6ec37127510ce09a4546d6fee6f Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 3 Jan 2009 07:36:35 +0000 Subject: [PATCH] updated default_settings with robots.txt related settings --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40631 --- scrapy/trunk/scrapy/conf/default_settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/trunk/scrapy/conf/default_settings.py b/scrapy/trunk/scrapy/conf/default_settings.py index 87de94fde..0d861297a 100644 --- a/scrapy/trunk/scrapy/conf/default_settings.py +++ b/scrapy/trunk/scrapy/conf/default_settings.py @@ -54,6 +54,7 @@ DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOADER_MIDDLEWARES = [ # Engine side + 'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware', 'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware', 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware', 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware', @@ -134,6 +135,8 @@ REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200'] +ROBOTSTXT_OBEY = True + SCHEDULER = 'scrapy.core.scheduler.Scheduler' SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO