From 0192282d07e95159282efe7af2ff155857a97550 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Tue, 18 Aug 2009 14:36:18 -0300
Subject: [PATCH] reorganized doc and moved robotstxt doc inside downloader
 middlewares doc

---
 docs/_templates/layout.html           | 11 +++--
 docs/experimental/index.rst           |  1 +
 docs/topics/downloader-middleware.rst | 19 +++++++++
 docs/topics/index.rst                 | 59 +++++++++++++++++++++++----
 docs/topics/robotstxt.rst             | 20 ---------
 5 files changed, 75 insertions(+), 35 deletions(-)
 delete mode 100644 docs/topics/robotstxt.rst

diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html
index a75908edc..6260e569a 100644
--- a/docs/_templates/layout.html
+++ b/docs/_templates/layout.html
@@ -2,12 +2,11 @@
 
 {% block rootrellink %}
         Scrapy v. {{ version }} documentation &raquo;
-    <li><a href="{{ pathto('index') }}">Home</a> | </li>
-        <li><a href="{{ pathto('intro/index') }}">Getting Started</a> | </li>
-        <li><a href="{{ pathto('topics/index') }}">Using Scrapy</a> | </li>
-        <li><a href="{{ pathto('reference') }}">API reference</a>  | </li>
-        <li><a href="{{ pathto('faq') }}">FAQ</a>  | </li>
-        <li><a href="{{ pathto('search') }}">Search</a> </li>
+    <li><a href="{{ pathto('index') }}">Start</a> | </li>
+        <li><a href="{{ pathto('intro/index') }}">Intro</a> | </li>
+        <li><a href="{{ pathto('topics/index') }}">Topics</a> | </li>
+        <li><a href="{{ pathto('reference') }}">Reference</a> | </li>
+        <li><a href="{{ pathto('faq') }}">FAQ</a> </li>
      
 {% endblock %}
 
diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst
index 8f4e33489..435a4695a 100644
--- a/docs/experimental/index.rst
+++ b/docs/experimental/index.rst
@@ -22,3 +22,4 @@ it's properly merged) . Use at your own risk.
    newitems
    loaders
    exporters
+   images
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 9ea0a059f..e8cd8158b 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -196,3 +196,22 @@ HttpCacheMiddleware
         * :setting:`HTTPCACHE_EXPIRATION_SECS` - how many secs until the cache is
           considered out of date
 
+RobotsTxtMiddleware
+-------------------
+
+.. module:: scrapy.contrib.downloadermiddleware.robotstxt
+   :synopsis: robots.txt middleware
+
+.. class:: RobotsTxtMiddleware:
+
+    This middleware filters out requests forbidden by the robots.txt exclusion
+    standard.
+
+    To make sure Scrapy respects robots.txt make sure the middleware is enabled
+    amd the :setting:`ROBOTSTXT_OBEY` setting is enabled.
+
+    .. warning:: Keep in mind that, if you crawl using multiple concurrent
+       requests per domain, Scrapy could still  download some forbidden pages
+       if they were requested before the robots.txt file was downloaded. This
+       is a known limitation of the current robots.txt middleware and will
+       be fixed in the future.
diff --git a/docs/topics/index.rst b/docs/topics/index.rst
index 49a115b96..500d7f612 100644
--- a/docs/topics/index.rst
+++ b/docs/topics/index.rst
@@ -5,6 +5,9 @@ Using Scrapy
 
 This section describes all key concepts of Scrapy.
 
+Chapter 1. Main topics
+----------------------
+
 .. toctree::
    :maxdepth: 1
 
@@ -12,22 +15,60 @@ This section describes all key concepts of Scrapy.
    shell
    spiders
    selectors
-   link-extractors
    items
+   request-response
+
+Chapter 2. Extension Mechanisms
+-------------------------------
+
+.. toctree::
+   :maxdepth: 1
+
    item-pipeline
    downloader-middleware
    spider-middleware
-   settings
+   scheduler-middleware
    extensions
+
+Chapter 3. Services
+-------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   settings
+   logging
+   signals
    stats
+   email
+
+Chapter 4. Administration consoles
+----------------------------------
+
+.. toctree::
+   :maxdepth: 1
+
    webconsole
    telnetconsole
-   robotstxt
+
+.. toctree::
+   :maxdepth: 1
+
+Chapter 5. Reference
+--------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   exceptions
+   link-extractors
+
+Appendix 1: Third party tools
+-----------------------------
+
+.. toctree::
+   :maxdepth: 1
+
    firefox
    firebug
-   signals
-   logging
-   scheduler-middleware
-   request-response
-   exceptions
-   email
+
diff --git a/docs/topics/robotstxt.rst b/docs/topics/robotstxt.rst
deleted file mode 100644
index 57b7718ac..000000000
--- a/docs/topics/robotstxt.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. _topics-robotstxt:
-
-==========
-robots.txt
-==========
-
-Scrapy deals with robots.txt files using a :ref:`topics-downloader-middleware`.
-called `RobotsTxtMiddleware`.
-
-To make sure Scrapy respects robots.txt files make sure the following
-middleware is enabled::
-
-     scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware
-
-And the :setting:`ROBOTSTXT_OBEY` setting is enabled.
-
-Keep in mind that, if you crawl using multiple concurrent requests per domain,
-Scrapy could get to download some forbidden pages if they were requested to
-download before the robots.txt file was downloaded. This is a known limitation
-of the current robots.txt middleware and will be fixed in the future.