diff --git a/scrapy/trunk/docs/ref/settings.rst b/scrapy/trunk/docs/ref/settings.rst index 002a8d128..f35461379 100644 --- a/scrapy/trunk/docs/ref/settings.rst +++ b/scrapy/trunk/docs/ref/settings.rst @@ -656,7 +656,7 @@ can be scheduled per domain. ROBOTSTXT_OBEY -------------- -Default: ``True`` +Default: ``False`` Scope: ``scrapy.contrib.downloadermiddleware.robotstxt`` diff --git a/scrapy/trunk/scrapy/conf/default_settings.py b/scrapy/trunk/scrapy/conf/default_settings.py index 334f6ff60..96c1220d6 100644 --- a/scrapy/trunk/scrapy/conf/default_settings.py +++ b/scrapy/trunk/scrapy/conf/default_settings.py @@ -143,7 +143,7 @@ REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200'] -ROBOTSTXT_OBEY = True +ROBOTSTXT_OBEY = False SCHEDULER = 'scrapy.core.scheduler.Scheduler' SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO