From 037e6c2125936c0cd8d0759e82850bbb69ed0f1d Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 1 Sep 2008 04:18:12 +0000 Subject: [PATCH] improved SpiderMiddleware's docstrings --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40192 --- scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py | 10 ++++++---- .../trunk/scrapy/contrib/spidermiddleware/offsite.py | 5 +++++ .../trunk/scrapy/contrib/spidermiddleware/referer.py | 6 +++++- .../trunk/scrapy/contrib/spidermiddleware/restrict.py | 2 +- .../trunk/scrapy/contrib/spidermiddleware/urllength.py | 6 ++++-- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py index 18fea0406..4d4d1bb8d 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py @@ -1,12 +1,14 @@ """ -Limits the scheduler request queue from the point of view of the spider. -That is, if the scheduler queue contains an equal or greater ammount of -requests than the specified limit, the further requests generated by the -spider will be ignored. +RequestLimitMiddleware: Limits the scheduler request queue from the point of +view of the spider. That is, if the scheduler queue contains an equal or +greater ammount of requests than the specified limit, the new requests +(generated by the spider) will be ignored. + The limit is setted from the spider attribute "requests_queue_size". If not found, from the scrapy setting "REQUESTS_QUEUE_SIZE". If not found, no limit will be applied. If given a value of 0, no limit will be applied. """ + from scrapy.core.engine import scrapyengine from scrapy.conf import settings from scrapy.http import Request diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py index 754d2276e..f3cc9c485 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py @@ -1,3 +1,8 @@ +""" +OffsiteMiddleware: Filters out Requests for URLs outside the domains covered by +the spider. +""" + from scrapy.core import log from scrapy.http import Request from scrapy.utils.url import url_is_from_spider diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py index 8d16093ed..ec2db9e9c 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py @@ -1,7 +1,11 @@ +""" +RefererMiddleware: populates Request referer field, based on the Response which +originated it. +""" + from scrapy.core import log from scrapy.http import Request - class CrawlMiddleware(object): def process_result(self, response, result, spider): def _set_referer(r): diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py index 22bd28fb3..b4eceef31 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py @@ -1,5 +1,5 @@ """ -Scrapemiddlware to restrict crawling to only some particular URLs +RestrictMiddleware: restricts crawling to fixed set of particular URLs """ from scrapy.http import Request diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py index 07dd8d5f3..a0ad4375a 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py @@ -1,11 +1,13 @@ +""" +UrlLengthMiddleware: Filters out requests with URLs longer than URLLENGTH_LIMIT +""" + from scrapy.core import log from scrapy.http import Request from scrapy.core.exceptions import NotConfigured from scrapy.conf import settings class UrlLengthMiddleware(object): - """This middleware discard requests with URLs longer than URLLENGTH_LIMIT""" - def __init__(self): self.maxlength = settings.getint('URLLENGTH_LIMIT') if not self.maxlength: