added scheduler request queue limit for spiders (spider middleware)

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40156

scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py

@@ -0,0 +1,35 @@
+"""
+Limits the scheduler request queue from the point of view of the spider.
+That is, any requests generated by the spider, which its scheduling action
+would result in an excess over the allowed limit of scheduler request queue,
+will be filtered out.
+The limit is setted from the spider attribute "requests_queue_size". If not
+found, from the scrapy setting "REQUESTS_QUEUE_SIZE". If not found, no limit
+will be applied. If given a value of 0, no limit will be applied.
+"""
+from scrapy.core.engine import scrapyengine
+from scrapy.conf import settings
+from scrapy.http import Request
+from scrapy.core import log
+
+class RequestLimitMiddleware(object):
+    _last_queue_size = 0
+    def process_result(self, response, result, spider):
+        requests = []
+        other = []
+        [requests.append(r) if isinstance(r, Request) else other.append(r) for r in result]
+
+        max_pending = spider.requests_queue_size if hasattr(spider,"requests_queue_size") else settings.getint("REQUESTS_QUEUE_SIZE")
+        if not max_pending:
+            accepted = requests
+        else:
+            free_slots = max_pending - len(scrapyengine.scheduler.pending_requests[spider.domain_name])
+            accepted = requests[:free_slots]
+            dropped = set(requests) - set(accepted)
+            if dropped:
+                for r in dropped:
+                    log.msg("Ignoring link (max schedule queue size reached): %s " % r.url, level=log.WARNING, domain=spider.domain_name)
+        actual_size = len(scrapyengine.scheduler.pending_requests[spider.domain_name])
+        log.msg("queue size: %d (%+d)" % (actual_size, actual_size - self._last_queue_size) )
+        self._last_queue_size = actual_size
+        return accepted + other

scrapy/trunk/scrapy/templates/settings.tmpl

@@ -52,6 +52,7 @@ DOWNLOADER_MIDDLEWARES = (
 
 SPIDER_MIDDLEWARES = (
     # Engine side
+    #'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware',
     #'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware',
     #'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware',
     #'scrapy.contrib.spidermiddleware.referer.CrawlMiddleware',