From 16f9f5a9ef264c840c390066d0cfdaf03ae23c03 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Sat, 3 Jan 2009 07:41:43 +0000
Subject: [PATCH] doc: added topic about robots.txt, added ROBOTSTXT_OBEY
 setting, added missing REQUESTS_PER_DOMAIN setting

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40633
---
 scrapy/trunk/docs/ref/settings.rst     | 24 +++++++++++++++++++++++-
 scrapy/trunk/docs/topics/index.rst     |  1 +
 scrapy/trunk/docs/topics/robotstxt.rst | 18 ++++++++++++++++++
 3 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 scrapy/trunk/docs/topics/robotstxt.rst

diff --git a/scrapy/trunk/docs/ref/settings.rst b/scrapy/trunk/docs/ref/settings.rst
index d3a0f091b..2620f29ae 100644
--- a/scrapy/trunk/docs/ref/settings.rst
+++ b/scrapy/trunk/docs/ref/settings.rst
@@ -268,6 +268,7 @@ DOWNLOADER_MIDDLEWARES
 Default:: 
 
     [
+        'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware',
         'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware',
         'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware',
         'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware',
@@ -519,6 +520,7 @@ MYSQL_CONNECTION_SETTINGS
 -------------------------
 
 Default: ``{}``
+
 Scope: ``scrapy.utils.db.mysql_connect``
 
 Settings to use for MySQL connections performed through
@@ -531,7 +533,7 @@ NEWSPIDER_MODULE
 
 Default: ``''``
 
-Module where to create new spiders using the genspider command.
+Module where to create new spiders using the ``genspider`` command.
 
 Example::
 
@@ -539,6 +541,14 @@ Example::
 
 .. setting:: REQUESTS_QUEUE_SIZE
 
+REQUESTS_PER_DOMAIN
+-------------------
+
+Default: ``8``
+
+Specifies how many concurrent (ie. simultaneous) requests will be performed per
+open spider.
+
 REQUESTS_QUEUE_SIZE
 -------------------
 
@@ -549,6 +559,18 @@ Scope: ``scrapy.contrib.spidermiddleware.limit``
 If non zero, it will be used as an upper limit for the amount of requests that
 can be scheduled per domain.
 
+.. setting:: ROBOTSTXT_OBEY
+
+ROBOTSTXT_OBEY
+--------------
+
+Default: ``True``
+
+Scope: ``scrapy.contrib.downloadermiddleware.robotstxt``
+
+If enabled, Scrapy will respect robots.txt policies. For more information see
+:topic:`robotstxt`
+
 .. setting:: SCHEDULER
 
 SCHEDULER
diff --git a/scrapy/trunk/docs/topics/index.rst b/scrapy/trunk/docs/topics/index.rst
index c6a3c37dd..36d397e60 100644
--- a/scrapy/trunk/docs/topics/index.rst
+++ b/scrapy/trunk/docs/topics/index.rst
@@ -9,3 +9,4 @@ Topics
    selectors
    settings
    spiders
+   robotstxt
diff --git a/scrapy/trunk/docs/topics/robotstxt.rst b/scrapy/trunk/docs/topics/robotstxt.rst
new file mode 100644
index 000000000..2d7cf5ad4
--- /dev/null
+++ b/scrapy/trunk/docs/topics/robotstxt.rst
@@ -0,0 +1,18 @@
+==================
+Obeying robots.txt
+==================
+
+Scrapy deals with robots.txt files using a :topic:`downloader-middleware`
+called `RobotsTxtMiddleware`.
+
+To make sure Scrapy respects robots.txt files make sure the following
+middleware is enabled::
+
+     scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware
+
+And the :setting:`ROBOTSTXT_OBEY` setting is enabled.
+
+Keep in mind that, if you crawl using multiple concurrent requests per domain,
+Scrapy could get to download some forbidden pages if they were requested to
+download before the robots.txt file was downloaded. This is a known limitation
+of the current robots.txt middleware and will be fixed in the future.