From aa345e116a6bde6e7e407576066bc1045c59235b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 21 Jul 2009 17:19:19 -0300 Subject: [PATCH] Added spider middleware documentation --- docs/ref/index.rst | 1 + docs/ref/settings.rst | 6 +- docs/ref/spider-middleware.rst | 111 ++++++++++++++++++ scrapy/contrib/spidermiddleware/depth.py | 21 ++-- scrapy/contrib/spidermiddleware/httperror.py | 11 +- scrapy/contrib/spidermiddleware/offsite.py | 5 +- .../contrib/spidermiddleware/requestlimit.py | 8 +- scrapy/contrib/spidermiddleware/restrict.py | 13 +- scrapy/contrib/spidermiddleware/urlfilter.py | 4 +- scrapy/contrib/spidermiddleware/urllength.py | 7 +- 10 files changed, 154 insertions(+), 33 deletions(-) create mode 100644 docs/ref/spider-middleware.rst diff --git a/docs/ref/index.rst b/docs/ref/index.rst index 0c394fb51..eb0adafde 100644 --- a/docs/ref/index.rst +++ b/docs/ref/index.rst @@ -19,6 +19,7 @@ This section documents the Scrapy |version| API. For more information see :ref:` extension-manager extensions downloader-middleware + spider-middleware scheduler-middleware link-extractors diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index 264b6e8a7..09dfc5098 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -680,6 +680,8 @@ Default: ``100`` Some sites use meta-refresh for redirecting to a session expired page, so we restrict automatic redirection to a maximum delay (in seconds) +.. setting:: REDIRECT_PRIORITY_ADJUST + REDIRECT_PRIORITY_ADJUST ------------------------------ @@ -688,7 +690,7 @@ Default: ``+2`` Adjust redirect request priority relative to original request. A negative priority adjust means more priority. -.. setting:: REQUESTS_QUEUE_SIZE +.. setting:: REQUESTS_PER_DOMAIN REQUESTS_PER_DOMAIN ------------------- @@ -698,6 +700,8 @@ Default: ``8`` Specifies how many concurrent (ie. simultaneous) requests will be performed per open spider. +.. setting:: REQUESTS_QUEUE_SIZE + REQUESTS_QUEUE_SIZE ------------------- diff --git a/docs/ref/spider-middleware.rst b/docs/ref/spider-middleware.rst new file mode 100644 index 000000000..e2497b953 --- /dev/null +++ b/docs/ref/spider-middleware.rst @@ -0,0 +1,111 @@ +.. _ref-spider-middleware: + +==================================== +Built-in spider middleware reference +==================================== + +This page describes all spider middleware components that come with Scrapy. For +information on how to use them and how to write your own spider middleware, see +the :ref:`spider middleware usage guide `. + +For a list of the components enabled by default (and their orders) see the +:setting:`SPIDER_MIDDLEWARES_BASE` setting. + +Available spider middlewares +============================ + +DepthMiddleware +--------------- + +.. module:: scrapy.contrib.spidermiddleware.depth + +.. class:: DepthMiddleware + + DepthMiddleware is a scrape middleware used for tracking the depth of each + Request inside the site being scraped. It can be used to limit the maximum + depth to scrape or things like that. + + The :class:`DepthMiddleware` can be configured through the following + settings (see the settings documentation for more info): + + * :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to + crawl for any site. If zero, no limit will be imposed. + * :setting:`DEPTH_STATS` - Whether to collect depth stats. + +HttpErrorMiddleware +------------------- + +.. module:: scrapy.contrib.spidermiddleware.httperror + +.. class:: HttpErrorMiddleware + + Filter out response outside of a range of valid status codes. + + This middleware filters out every response with status outside of the range + 200<=status<300. Spiders can add more exceptions using + ``handle_httpstatus_list`` spider attribute. + +OffsiteMiddleware +----------------- + +.. module:: scrapy.contrib.spidermiddleware.offsite + +.. class:: OffsiteMiddleware + + Filters out Requests for URLs outside the domains covered by the spider. + +RequestLimitMiddleware +---------------------- + +.. module:: scrapy.contrib.spidermiddleware.requestlimit + +.. class:: RequestLimitMiddleware + + Limits the maximum number of requests in the scheduler for each spider. When + a spider tries to schedule more than the allowed amount of requests, the new + requests (returned by the spider) will be dropped. + + The :class:`RequestLimitMiddleware` can be configured through the following + settings (see the settings documentation for more info): + + * :setting:`REQUESTS_QUEUE_SIZE` - If non zero, it will be used as an + upper limit for the amount of requests that can be scheduled per + domain. Can be set per spider using ``requests_queue_size`` attribute. + +RestrictMiddleware +------------------ + +.. module:: scrapy.contrib.spidermiddleware.restrict + +.. class:: RestrictMiddleware + + Restricts crawling to fixed set of particular URLs. + + The :class:`RestrictMiddleware` can be configured through the following + settings (see the settings documentation for more info): + + * :setting:`RESTRICT_TO_URLS` - Set of URLs allowed to crawl. + +UrlFilterMiddleware +------------------- + +.. module:: scrapy.contrib.spidermiddleware.urlfilter + +.. class:: UrlFilterMiddleware + + Canonicalizes URLs to filter out duplicated ones + +UrlLengthMiddleware +------------------- + +.. module:: scrapy.contrib.spidermiddleware.urllength + +.. class:: UrlLengthMiddleware + + Filters out requests with URLs longer than URLLENGTH_LIMIT + + The :class:`UrlLengthMiddleware` can be configured through the following + settings (see the settings documentation for more info): + + * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. + diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py index d201d18d4..f91da1039 100644 --- a/scrapy/contrib/spidermiddleware/depth.py +++ b/scrapy/contrib/spidermiddleware/depth.py @@ -1,7 +1,7 @@ """ -DepthMiddleware is a scrape middleware used for tracking the depth of each -Request inside the site being scraped. It can be used to limit the maximum -depth to scrape or things like that +Depth Spider Middleware + +See documentation in docs/ref/spider-middleware.rst """ from scrapy import log @@ -18,21 +18,24 @@ class DepthMiddleware(object): stats.set_value('envinfo/request_depth_limit', self.maxdepth) def process_spider_output(self, response, result, spider): + domain = spider.domain_name def _filter(request): if isinstance(request, Request): depth = response.request.meta['depth'] + 1 request.meta['depth'] = depth if self.maxdepth and depth > self.maxdepth: - log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), level=log.DEBUG, domain=spider.domain_name) + log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), \ + level=log.DEBUG, domain=domain) return False elif self.stats: - stats.inc_value('request_depth_count/%s' % depth, domain=spider.domain_name) - if depth > stats.get_value('request_depth_max', 0, domain=spider.domain_name): - stats.set_value('request_depth_max', depth, domain=spider.domain_name) + stats.inc_value('request_depth_count/%s' % depth, domain=domain) + if depth > stats.get_value('request_depth_max', 0, domain=domain): + stats.set_value('request_depth_max', depth, domain=domain) return True - if self.stats and 'depth' not in response.request.meta: # otherwise we loose stats for depth=0 + # base case (depth=0) + if self.stats and 'depth' not in response.request.meta: response.request.meta['depth'] = 0 - stats.inc_value('request_depth_count/0', domain=spider.domain_name) + stats.inc_value('request_depth_count/0', domain=domain) return (r for r in result or () if _filter(r)) diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py index 77c7b36f4..4031f3e16 100644 --- a/scrapy/contrib/spidermiddleware/httperror.py +++ b/scrapy/contrib/spidermiddleware/httperror.py @@ -1,13 +1,12 @@ +""" +HttpError Spider Middleware +See documentation in docs/ref/spider-middleware.rst +""" class HttpErrorMiddleware(object): - """Filter out response outside of a range of valid status codes - - This middleware filters out every response with status outside of the range 200<=status<300 - Spiders can add more exceptions using `handle_httpstatus_list` spider attribute. - """ def process_spider_input(self, response, spider): if not (200 <= response.status < 300 or \ response.status in getattr(spider, 'handle_httpstatus_list', [])): - return [] # skip response + return [] diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index e6b233aa2..14375d9fc 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -1,6 +1,7 @@ """ -OffsiteMiddleware: Filters out Requests for URLs outside the domains covered by -the spider. +Offsite Spider Middleware + +See documentation in docs/ref/spider-middleware.rst """ from scrapy.http import Request diff --git a/scrapy/contrib/spidermiddleware/requestlimit.py b/scrapy/contrib/spidermiddleware/requestlimit.py index 73779ae55..edbd9a9c7 100644 --- a/scrapy/contrib/spidermiddleware/requestlimit.py +++ b/scrapy/contrib/spidermiddleware/requestlimit.py @@ -1,11 +1,7 @@ """ -RequestLimitMiddleware: Limits the scheduler request queue size. When spiders -try to schedule more than the allowed amount of requests the new requests -(returned by the spider) will be dropped. +Request Limit Spider middleware -The limit can be set using the spider attribue `requests_queue_size` or the -setting "REQUESTS_QUEUE_SIZE". If not specified (or 0), no limit will be -applied. +See documentation in docs/ref/spider-middleware.rst """ from itertools import imap from scrapy.xlib.pydispatch import dispatcher diff --git a/scrapy/contrib/spidermiddleware/restrict.py b/scrapy/contrib/spidermiddleware/restrict.py index 5b01d7be7..a48ebfe7f 100644 --- a/scrapy/contrib/spidermiddleware/restrict.py +++ b/scrapy/contrib/spidermiddleware/restrict.py @@ -1,7 +1,11 @@ """ -RestrictMiddleware: restricts crawling to fixed set of particular URLs +Restrict Spider Middleware + +See documentation in docs/ref/spider-middleware.rst """ +from itertools import ifilter + from scrapy.http import Request from scrapy.core.exceptions import NotConfigured from scrapy.conf import settings @@ -13,9 +17,6 @@ class RestrictMiddleware(object): raise NotConfigured def process_spider_output(self, response, result, spider): - def _filter(r): - if isinstance(r, Request) and r.url not in self.allowed_urls: - return False - return True - return (r for r in result or () if _filter(r)) + return ifilter(lambda r: isinstance(r, Request) \ + and r.url not in self.allowed_urls, result or ()) diff --git a/scrapy/contrib/spidermiddleware/urlfilter.py b/scrapy/contrib/spidermiddleware/urlfilter.py index e94c0c52e..e3d30eb6e 100644 --- a/scrapy/contrib/spidermiddleware/urlfilter.py +++ b/scrapy/contrib/spidermiddleware/urlfilter.py @@ -1,5 +1,7 @@ """ -UrlFilterMiddleware: canonicalizes URLs to filter out duplicated ones +Url Filter Middleware + +See documentation in docs/refs/spider-middleware.rst """ from scrapy.http import Request diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py index 600904ce0..cbdb022a6 100644 --- a/scrapy/contrib/spidermiddleware/urllength.py +++ b/scrapy/contrib/spidermiddleware/urllength.py @@ -1,5 +1,7 @@ """ -UrlLengthMiddleware: Filters out requests with URLs longer than URLLENGTH_LIMIT +Url Length Spider Middleware + +See documentation in docs/ref/spider-middleware.rst """ from scrapy import log @@ -16,7 +18,8 @@ class UrlLengthMiddleware(object): def process_spider_output(self, response, result, spider): def _filter(request): if isinstance(request, Request) and len(request.url) > self.maxlength: - log.msg("Ignoring link (url length > %d): %s " % (self.maxlength, request.url), level=log.DEBUG, domain=spider.domain_name) + log.msg("Ignoring link (url length > %d): %s " % (self.maxlength, request.url), \ + level=log.DEBUG, domain=spider.domain_name) return False else: return True