From a690d33f244e4ff1e504afeaf4c11bcd25b86d9e Mon Sep 17 00:00:00 2001 From: olveyra Date: Tue, 12 Aug 2008 15:11:15 +0000 Subject: [PATCH] added scheduler request queue limit for spiders (spider middleware) --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40156 --- .../scrapy/contrib/spidermiddleware/limit.py | 35 +++++++++++++++++++ scrapy/trunk/scrapy/templates/settings.tmpl | 1 + 2 files changed, 36 insertions(+) create mode 100644 scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py new file mode 100644 index 000000000..0f78e995c --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py @@ -0,0 +1,35 @@ +""" +Limits the scheduler request queue from the point of view of the spider. +That is, any requests generated by the spider, which its scheduling action +would result in an excess over the allowed limit of scheduler request queue, +will be filtered out. +The limit is setted from the spider attribute "requests_queue_size". If not +found, from the scrapy setting "REQUESTS_QUEUE_SIZE". If not found, no limit +will be applied. If given a value of 0, no limit will be applied. +""" +from scrapy.core.engine import scrapyengine +from scrapy.conf import settings +from scrapy.http import Request +from scrapy.core import log + +class RequestLimitMiddleware(object): + _last_queue_size = 0 + def process_result(self, response, result, spider): + requests = [] + other = [] + [requests.append(r) if isinstance(r, Request) else other.append(r) for r in result] + + max_pending = spider.requests_queue_size if hasattr(spider,"requests_queue_size") else settings.getint("REQUESTS_QUEUE_SIZE") + if not max_pending: + accepted = requests + else: + free_slots = max_pending - len(scrapyengine.scheduler.pending_requests[spider.domain_name]) + accepted = requests[:free_slots] + dropped = set(requests) - set(accepted) + if dropped: + for r in dropped: + log.msg("Ignoring link (max schedule queue size reached): %s " % r.url, level=log.WARNING, domain=spider.domain_name) + actual_size = len(scrapyengine.scheduler.pending_requests[spider.domain_name]) + log.msg("queue size: %d (%+d)" % (actual_size, actual_size - self._last_queue_size) ) + self._last_queue_size = actual_size + return accepted + other diff --git a/scrapy/trunk/scrapy/templates/settings.tmpl b/scrapy/trunk/scrapy/templates/settings.tmpl index 7ca6bfddd..9a9576da6 100644 --- a/scrapy/trunk/scrapy/templates/settings.tmpl +++ b/scrapy/trunk/scrapy/templates/settings.tmpl @@ -52,6 +52,7 @@ DOWNLOADER_MIDDLEWARES = ( SPIDER_MIDDLEWARES = ( # Engine side + #'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware', #'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware', #'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware', #'scrapy.contrib.spidermiddleware.referer.CrawlMiddleware',