From f4e6ed3c2fcf92b37a907279dc31c97d1bf27c3b Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 29 Jan 2009 18:22:59 +0000 Subject: [PATCH] run without RobotTxtMiddleware by default and update docs --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40796 --- scrapy/trunk/docs/ref/settings.rst | 2 +- scrapy/trunk/scrapy/conf/default_settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/trunk/docs/ref/settings.rst b/scrapy/trunk/docs/ref/settings.rst index 002a8d128..f35461379 100644 --- a/scrapy/trunk/docs/ref/settings.rst +++ b/scrapy/trunk/docs/ref/settings.rst @@ -656,7 +656,7 @@ can be scheduled per domain. ROBOTSTXT_OBEY -------------- -Default: ``True`` +Default: ``False`` Scope: ``scrapy.contrib.downloadermiddleware.robotstxt`` diff --git a/scrapy/trunk/scrapy/conf/default_settings.py b/scrapy/trunk/scrapy/conf/default_settings.py index 334f6ff60..96c1220d6 100644 --- a/scrapy/trunk/scrapy/conf/default_settings.py +++ b/scrapy/trunk/scrapy/conf/default_settings.py @@ -143,7 +143,7 @@ REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200'] -ROBOTSTXT_OBEY = True +ROBOTSTXT_OBEY = False SCHEDULER = 'scrapy.core.scheduler.Scheduler' SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO