Added spider middleware documentation

docs/ref/index.rst

@@ -19,6 +19,7 @@ This section documents the Scrapy |version| API. For more information see :ref:`
    extension-manager
    extensions
    downloader-middleware
+   spider-middleware
    scheduler-middleware
    link-extractors
 

docs/ref/settings.rst

@@ -680,6 +680,8 @@ Default: ``100``
 Some sites use meta-refresh for redirecting to a session expired page, so we
 restrict automatic redirection to a maximum delay (in seconds)
 
+.. setting:: REDIRECT_PRIORITY_ADJUST
+
 REDIRECT_PRIORITY_ADJUST
 ------------------------------
 
@@ -688,7 +690,7 @@ Default: ``+2``
 Adjust redirect request priority relative to original request.
 A negative priority adjust means more priority.
 
-.. setting:: REQUESTS_QUEUE_SIZE
+.. setting:: REQUESTS_PER_DOMAIN
 
 REQUESTS_PER_DOMAIN
 -------------------
@@ -698,6 +700,8 @@ Default: ``8``
 Specifies how many concurrent (ie. simultaneous) requests will be performed per
 open spider.
 
+.. setting:: REQUESTS_QUEUE_SIZE
+
 REQUESTS_QUEUE_SIZE
 -------------------
 

docs/ref/spider-middleware.rst

@@ -0,0 +1,111 @@
+.. _ref-spider-middleware:
+
+====================================
+Built-in spider middleware reference
+====================================
+
+This page describes all spider middleware components that come with Scrapy. For
+information on how to use them and how to write your own spider middleware, see
+the :ref:`spider middleware usage guide <topics-spider-middleware>`.
+
+For a list of the components enabled by default (and their orders) see the
+:setting:`SPIDER_MIDDLEWARES_BASE` setting.
+
+Available spider middlewares
+============================
+
+DepthMiddleware
+---------------
+
+.. module:: scrapy.contrib.spidermiddleware.depth
+
+.. class:: DepthMiddleware
+
+   DepthMiddleware is a scrape middleware used for tracking the depth of each
+   Request inside the site being scraped. It can be used to limit the maximum
+   depth to scrape or things like that.
+
+   The :class:`DepthMiddleware` can be configured through the following
+   settings (see the settings documentation for more info):
+
+      * :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to
+        crawl for any site. If zero, no limit will be imposed.
+      * :setting:`DEPTH_STATS` - Whether to collect depth stats.
+
+HttpErrorMiddleware
+-------------------
+
+.. module:: scrapy.contrib.spidermiddleware.httperror
+
+.. class:: HttpErrorMiddleware
+
+    Filter out response outside of a range of valid status codes.
+
+    This middleware filters out every response with status outside of the range
+    200<=status<300. Spiders can add more exceptions using
+    ``handle_httpstatus_list`` spider attribute.
+
+OffsiteMiddleware
+-----------------
+
+.. module:: scrapy.contrib.spidermiddleware.offsite
+
+.. class:: OffsiteMiddleware
+
+   Filters out Requests for URLs outside the domains covered by the spider.
+
+RequestLimitMiddleware
+----------------------
+
+.. module:: scrapy.contrib.spidermiddleware.requestlimit
+
+.. class:: RequestLimitMiddleware
+
+   Limits the maximum number of requests in the scheduler for each spider. When
+   a spider tries to schedule more than the allowed amount of requests, the new
+   requests (returned by the spider) will be dropped.
+
+   The :class:`RequestLimitMiddleware` can be configured through the following
+   settings (see the settings documentation for more info):
+
+      * :setting:`REQUESTS_QUEUE_SIZE` - If non zero, it will be used as an
+        upper limit for the amount of requests that can be scheduled per
+        domain. Can be set per spider using ``requests_queue_size`` attribute.
+
+RestrictMiddleware
+------------------
+
+.. module:: scrapy.contrib.spidermiddleware.restrict
+
+.. class:: RestrictMiddleware 
+
+   Restricts crawling to fixed set of particular URLs.
+
+   The :class:`RestrictMiddleware` can be configured through the following
+   settings (see the settings documentation for more info):
+
+      * :setting:`RESTRICT_TO_URLS` - Set of URLs allowed to crawl.
+
+UrlFilterMiddleware
+-------------------
+
+.. module:: scrapy.contrib.spidermiddleware.urlfilter
+
+.. class:: UrlFilterMiddleware 
+
+   Canonicalizes URLs to filter out duplicated ones
+
+UrlLengthMiddleware
+-------------------
+
+.. module:: scrapy.contrib.spidermiddleware.urllength
+
+.. class:: UrlLengthMiddleware 
+
+   Filters out requests with URLs longer than URLLENGTH_LIMIT
+
+   The :class:`UrlLengthMiddleware` can be configured through the following
+   settings (see the settings documentation for more info):
+
+      * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.
+

scrapy/contrib/spidermiddleware/depth.py

@@ -1,7 +1,7 @@
 """
-DepthMiddleware is a scrape middleware used for tracking the depth of each
-Request inside the site being scraped. It can be used to limit the maximum
-depth to scrape or things like that
+Depth Spider Middleware
+
+See documentation in docs/ref/spider-middleware.rst
 """
 
 from scrapy import log
@@ -18,21 +18,24 @@ class DepthMiddleware(object):
             stats.set_value('envinfo/request_depth_limit', self.maxdepth)
 
     def process_spider_output(self, response, result, spider):
+        domain = spider.domain_name
         def _filter(request):
             if isinstance(request, Request):
                 depth = response.request.meta['depth'] + 1
                 request.meta['depth'] = depth
                 if self.maxdepth and depth > self.maxdepth:
-                    log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), level=log.DEBUG, domain=spider.domain_name)
+                    log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), \
+                        level=log.DEBUG, domain=domain)
                     return False
                 elif self.stats:
-                    stats.inc_value('request_depth_count/%s' % depth, domain=spider.domain_name)
-                    if depth > stats.get_value('request_depth_max', 0, domain=spider.domain_name):
-                        stats.set_value('request_depth_max', depth, domain=spider.domain_name)
+                    stats.inc_value('request_depth_count/%s' % depth, domain=domain)
+                    if depth > stats.get_value('request_depth_max', 0, domain=domain):
+                        stats.set_value('request_depth_max', depth, domain=domain)
             return True
 
-        if self.stats and 'depth' not in response.request.meta: # otherwise we loose stats for depth=0 
+        # base case (depth=0)
+        if self.stats and 'depth' not in response.request.meta: 
             response.request.meta['depth'] = 0
-            stats.inc_value('request_depth_count/0', domain=spider.domain_name)
+            stats.inc_value('request_depth_count/0', domain=domain)
 
         return (r for r in result or () if _filter(r))

scrapy/contrib/spidermiddleware/httperror.py

@@ -1,13 +1,12 @@
+"""
+HttpError Spider Middleware
 
+See documentation in docs/ref/spider-middleware.rst
+"""
 
 class HttpErrorMiddleware(object):
-    """Filter out response outside of a range of valid status codes
-
-    This middleware filters out every response with status outside of the range 200<=status<300
-    Spiders can add more exceptions using `handle_httpstatus_list` spider attribute.
-    """
 
     def process_spider_input(self, response, spider):
         if not (200 <= response.status < 300 or \
                 response.status in getattr(spider, 'handle_httpstatus_list', [])):
-            return [] # skip response
+            return []

scrapy/contrib/spidermiddleware/offsite.py

@@ -1,6 +1,7 @@
 """
-OffsiteMiddleware: Filters out Requests for URLs outside the domains covered by
-the spider.
+Offsite Spider Middleware
+
+See documentation in docs/ref/spider-middleware.rst
 """
 
 from scrapy.http import Request

scrapy/contrib/spidermiddleware/requestlimit.py

@@ -1,11 +1,7 @@
 """
-RequestLimitMiddleware: Limits the scheduler request queue size. When spiders
-try to schedule more than the allowed amount of requests the new requests
-(returned by the spider) will be dropped.
+Request Limit Spider middleware
 
-The limit can be set using the spider attribue `requests_queue_size` or the
-setting "REQUESTS_QUEUE_SIZE". If not specified (or 0), no limit will be
-applied. 
+See documentation in docs/ref/spider-middleware.rst
 """
 from itertools import imap
 from scrapy.xlib.pydispatch import dispatcher

scrapy/contrib/spidermiddleware/restrict.py

@@ -1,7 +1,11 @@
 """
-RestrictMiddleware: restricts crawling to fixed set of particular URLs
+Restrict Spider Middleware
+
+See documentation in docs/ref/spider-middleware.rst
 """
 
+from itertools import ifilter
+
 from scrapy.http import Request
 from scrapy.core.exceptions import NotConfigured
 from scrapy.conf import settings
@@ -13,9 +17,6 @@ class RestrictMiddleware(object):
             raise NotConfigured
 
     def process_spider_output(self, response, result, spider):
-        def _filter(r):
-            if isinstance(r, Request) and r.url not in self.allowed_urls:
-                return False
-            return True
-        return (r for r in result or () if _filter(r))
+        return ifilter(lambda r: isinstance(r, Request) \
+            and r.url not in self.allowed_urls, result or ())
 

scrapy/contrib/spidermiddleware/urlfilter.py

@@ -1,5 +1,7 @@
 """
-UrlFilterMiddleware: canonicalizes URLs to filter out duplicated ones
+Url Filter Middleware
+
+See documentation in docs/refs/spider-middleware.rst
 """
 
 from scrapy.http import Request

scrapy/contrib/spidermiddleware/urllength.py

@@ -1,5 +1,7 @@
 """
-UrlLengthMiddleware: Filters out requests with URLs longer than URLLENGTH_LIMIT
+Url Length Spider Middleware
+
+See documentation in docs/ref/spider-middleware.rst
 """
 
 from scrapy import log
@@ -16,7 +18,8 @@ class UrlLengthMiddleware(object):
     def process_spider_output(self, response, result, spider):
         def _filter(request):
             if isinstance(request, Request) and len(request.url) > self.maxlength:
-                log.msg("Ignoring link (url length > %d): %s " % (self.maxlength, request.url), level=log.DEBUG, domain=spider.domain_name)
+                log.msg("Ignoring link (url length > %d): %s " % (self.maxlength, request.url), \
+                    level=log.DEBUG, domain=spider.domain_name)
                 return False
             else:
                 return True