diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e3b452419..0f2f99811 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -768,18 +768,6 @@ Default: ``+2`` Adjust redirect request priority relative to original request. A negative priority adjust means more priority. -.. setting:: REQUESTS_QUEUE_SIZE - -REQUESTS_QUEUE_SIZE -------------------- - -Default: ``0`` - -Scope: ``scrapy.contrib.spidermiddleware.limit`` - -If non zero, it will be used as an upper limit for the amount of requests that -can be scheduled per domain. - .. setting:: ROBOTSTXT_OBEY ROBOTSTXT_OBEY @@ -866,7 +854,6 @@ Default:: { 'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50, 'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100, - 'scrapy.contrib.spidermiddleware.requestlimit.RequestLimitMiddleware': 200, 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500, 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700, 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800, diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index fbbdccfe8..1db93259a 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -245,25 +245,6 @@ RefererMiddleware Populates Request referer field, based on the Response which originated it. -RequestLimitMiddleware ----------------------- - -.. module:: scrapy.contrib.spidermiddleware.requestlimit - :synopsis: Request limit Spider Middleware - -.. class:: RequestLimitMiddleware - - Limits the maximum number of requests in the scheduler for each spider. When - a spider tries to schedule more than the allowed amount of requests, the new - requests (returned by the spider) will be dropped. - - The :class:`RequestLimitMiddleware` can be configured through the following - settings (see the settings documentation for more info): - - * :setting:`REQUESTS_QUEUE_SIZE` - If non zero, it will be used as an - upper limit for the amount of requests that can be scheduled per - domain. Can be set per spider using ``requests_queue_size`` attribute. - UrlLengthMiddleware ------------------- diff --git a/scrapy/contrib/spidermiddleware/requestlimit.py b/scrapy/contrib/spidermiddleware/requestlimit.py deleted file mode 100644 index bd52576f7..000000000 --- a/scrapy/contrib/spidermiddleware/requestlimit.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -Request Limit Spider middleware - -See documentation in docs/topics/spider-middleware.rst -""" -from itertools import imap -from scrapy.xlib.pydispatch import dispatcher - -from scrapy import signals -from scrapy.project import crawler -from scrapy.exceptions import NotConfigured -from scrapy.conf import settings -from scrapy.http import Request -from scrapy import log - -class RequestLimitMiddleware(object): - - def __init__(self): - self.max_queue_size = settings.getint("REQUESTS_QUEUE_SIZE") - if not self.max_queue_size: - raise NotConfigured - - self.max_pending = {} - self.dropped_count = {} - - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - - def spider_opened(self, spider): - self.max_pending[spider] = getattr(spider, 'requests_queue_size', self.max_queue_size) - self.dropped_count[spider] = 0 - - def spider_closed(self, spider): - dropped_count = self.dropped_count[spider] - if dropped_count: - max_pending = self.max_pending[spider] - log.msg('Dropped %d request(s) because the scheduler queue size limit (%d requests) was exceeded' % \ - (dropped_count, max_pending), level=log.DEBUG, spider=spider) - del self.dropped_count[spider] - del self.max_pending[spider] - - def process_spider_output(self, response, result, spider): - max_pending = self.max_pending.get(spider, 0) - if max_pending: - return imap(lambda v: self._limit_requests(v, spider, max_pending), result) - else: - return result - - def _limit_requests(self, request_or_other, spider, max_pending): - if isinstance(request_or_other, Request): - free_slots = max_pending - self._pending_count(spider) - if free_slots > 0: - # Scheduler isn't saturated and it is fine to schedule more requests. - return request_or_other - else: - # Skip the request and give engine time to handle other tasks. - self.dropped_count[spider] += 1 - return None - else: - # Return others (non-requests) as is. - return request_or_other - - def _pending_count(self, spider): - pending = crawler.engine.scheduler.pending_requests.get(spider, []) - return len(pending) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 724f24237..4f0071a96 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -194,8 +194,6 @@ REDIRECT_MAX_METAREFRESH_DELAY = 100 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 -REQUESTS_QUEUE_SIZE = 0 - # contrib.middleware.retry.RetryMiddleware default settings RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = ['500', '503', '504', '400', '408'] @@ -220,7 +218,6 @@ SPIDER_MIDDLEWARES = {} SPIDER_MIDDLEWARES_BASE = { # Engine side 'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50, - 'scrapy.contrib.spidermiddleware.requestlimit.RequestLimitMiddleware': 200, 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500, 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700, 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800,