RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

commented debug lines in last commit 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40157
This commit is contained in:
olveyra 2008-08-12 15:13:48 +00:00
parent a690d33f24
commit e3f70a8101
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py
View File

@ -5,7 +5,7 @@ would result in an excess over the allowed limit of scheduler request queue,
will be filtered out.
The limit is setted from the spider attribute "requests_queue_size". If not
found, from the scrapy setting "REQUESTS_QUEUE_SIZE". If not found, no limit
will be applied. If given a value of 0, no limit will be applied.
will be applied.　If given a value of 0, no limit will be applied.
"""
from scrapy.core.engine import scrapyengine
from scrapy.conf import settings
@ -13,7 +13,7 @@ from scrapy.http import Request
from scrapy.core import log
class RequestLimitMiddleware(object):
_last_queue_size = 0
#_last_queue_size = 0
def process_result(self, response, result, spider):
requests = []
other = []
@ -29,7 +29,7 @@ class RequestLimitMiddleware(object):
if dropped:
for r in dropped:
log.msg("Ignoring link (max schedule queue size reached): %s " % r.url, level=log.WARNING, domain=spider.domain_name)
actual_size = len(scrapyengine.scheduler.pending_requests[spider.domain_name])
log.msg("queue size: %d (%+d)" % (actual_size, actual_size - self._last_queue_size) )
self._last_queue_size = actual_size
#actual_size = len(scrapyengine.scheduler.pending_requests[spider.domain_name])
#log.msg("queue size: %d (%+d)" % (actual_size, actual_size - self._last_queue_size) )
#self._last_queue_size = actual_size
return accepted + other