mirror of https://github.com/scrapy/scrapy.git
Removed RequestLimitMiddleware
This commit is contained in:
Pablo Hoffman
parent
97d77c79c2
commit
9599bde3e9
|
|
@ -768,18 +768,6 @@ Default: ``+2``
|
|||
Adjust redirect request priority relative to original request.
|
||||
A negative priority adjust means more priority.
|
||||
|
||||
.. setting:: REQUESTS_QUEUE_SIZE
|
||||
|
||||
REQUESTS_QUEUE_SIZE
|
||||
-------------------
|
||||
|
||||
Default: ``0``
|
||||
|
||||
Scope: ``scrapy.contrib.spidermiddleware.limit``
|
||||
|
||||
If non zero, it will be used as an upper limit for the amount of requests that
|
||||
can be scheduled per domain.
|
||||
|
||||
.. setting:: ROBOTSTXT_OBEY
|
||||
|
||||
ROBOTSTXT_OBEY
|
||||
|
|
@ -866,7 +854,6 @@ Default::
|
|||
{
|
||||
'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50,
|
||||
'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100,
|
||||
'scrapy.contrib.spidermiddleware.requestlimit.RequestLimitMiddleware': 200,
|
||||
'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500,
|
||||
'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700,
|
||||
'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800,
|
||||
|
|
|
|||
|
|
@ -245,25 +245,6 @@ RefererMiddleware
|
|||
|
||||
Populates Request referer field, based on the Response which originated it.
|
||||
|
||||
RequestLimitMiddleware
|
||||
----------------------
|
||||
|
||||
.. module:: scrapy.contrib.spidermiddleware.requestlimit
|
||||
:synopsis: Request limit Spider Middleware
|
||||
|
||||
.. class:: RequestLimitMiddleware
|
||||
|
||||
Limits the maximum number of requests in the scheduler for each spider. When
|
||||
a spider tries to schedule more than the allowed amount of requests, the new
|
||||
requests (returned by the spider) will be dropped.
|
||||
|
||||
The :class:`RequestLimitMiddleware` can be configured through the following
|
||||
settings (see the settings documentation for more info):
|
||||
|
||||
* :setting:`REQUESTS_QUEUE_SIZE` - If non zero, it will be used as an
|
||||
upper limit for the amount of requests that can be scheduled per
|
||||
domain. Can be set per spider using ``requests_queue_size`` attribute.
|
||||
|
||||
UrlLengthMiddleware
|
||||
-------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -1,65 +0,0 @@
|
|||
"""
|
||||
Request Limit Spider middleware
|
||||
|
||||
See documentation in docs/topics/spider-middleware.rst
|
||||
"""
|
||||
from itertools import imap
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.project import crawler
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.conf import settings
|
||||
from scrapy.http import Request
|
||||
from scrapy import log
|
||||
|
||||
class RequestLimitMiddleware(object):
|
||||
|
||||
def __init__(self):
|
||||
self.max_queue_size = settings.getint("REQUESTS_QUEUE_SIZE")
|
||||
if not self.max_queue_size:
|
||||
raise NotConfigured
|
||||
|
||||
self.max_pending = {}
|
||||
self.dropped_count = {}
|
||||
|
||||
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
|
||||
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
|
||||
|
||||
def spider_opened(self, spider):
|
||||
self.max_pending[spider] = getattr(spider, 'requests_queue_size', self.max_queue_size)
|
||||
self.dropped_count[spider] = 0
|
||||
|
||||
def spider_closed(self, spider):
|
||||
dropped_count = self.dropped_count[spider]
|
||||
if dropped_count:
|
||||
max_pending = self.max_pending[spider]
|
||||
log.msg('Dropped %d request(s) because the scheduler queue size limit (%d requests) was exceeded' % \
|
||||
(dropped_count, max_pending), level=log.DEBUG, spider=spider)
|
||||
del self.dropped_count[spider]
|
||||
del self.max_pending[spider]
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
max_pending = self.max_pending.get(spider, 0)
|
||||
if max_pending:
|
||||
return imap(lambda v: self._limit_requests(v, spider, max_pending), result)
|
||||
else:
|
||||
return result
|
||||
|
||||
def _limit_requests(self, request_or_other, spider, max_pending):
|
||||
if isinstance(request_or_other, Request):
|
||||
free_slots = max_pending - self._pending_count(spider)
|
||||
if free_slots > 0:
|
||||
# Scheduler isn't saturated and it is fine to schedule more requests.
|
||||
return request_or_other
|
||||
else:
|
||||
# Skip the request and give engine time to handle other tasks.
|
||||
self.dropped_count[spider] += 1
|
||||
return None
|
||||
else:
|
||||
# Return others (non-requests) as is.
|
||||
return request_or_other
|
||||
|
||||
def _pending_count(self, spider):
|
||||
pending = crawler.engine.scheduler.pending_requests.get(spider, [])
|
||||
return len(pending)
|
||||
|
|
@ -194,8 +194,6 @@ REDIRECT_MAX_METAREFRESH_DELAY = 100
|
|||
REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
|
||||
REDIRECT_PRIORITY_ADJUST = +2
|
||||
|
||||
REQUESTS_QUEUE_SIZE = 0
|
||||
|
||||
# contrib.middleware.retry.RetryMiddleware default settings
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
RETRY_HTTP_CODES = ['500', '503', '504', '400', '408']
|
||||
|
|
@ -220,7 +218,6 @@ SPIDER_MIDDLEWARES = {}
|
|||
SPIDER_MIDDLEWARES_BASE = {
|
||||
# Engine side
|
||||
'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50,
|
||||
'scrapy.contrib.spidermiddleware.requestlimit.RequestLimitMiddleware': 200,
|
||||
'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500,
|
||||
'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700,
|
||||
'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800,
|
||||
|
|
|
|||
Loading…
Reference in New Issue