RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added more scheduler stats

This commit is contained in:
Pablo Hoffman 2012-09-13 19:46:45 -03:00
parent d6867f7984
commit 263e302b6a
1 changed files with 16 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
scrapy/core/scheduler.py
View File

@ -47,11 +47,25 @@ class Scheduler(object):
def enqueue_request(self, request):
if not request.dont_filter and self.df.request_seen(request):
return
if not self._dqpush(request):
dqok = self._dqpush(request)
if dqok:
self.stats.inc_value('scheduler/disk_enqueued', spider=self.spider)
else:
self._mqpush(request)
self.stats.inc_value('scheduler/memory_enqueued', spider=self.spider)
self.stats.inc_value('scheduler/enqueued', spider=self.spider)
def next_request(self):
return self.mqs.pop() or self._dqpop()
request = self.mqs.pop()
if request:
self.stats.inc_value('scheduler/memory_dequeued', spider=self.spider)
else:
request = self._dqpop()
if request:
self.stats.inc_value('scheduler/disk_dequeued', spider=self.spider)
if request:
self.stats.inc_value('scheduler/dequeued', spider=self.spider)
return request
def __len__(self):
return len(self.dqs) + len(self.mqs) if self.dqs else len(self.mqs)
@ -69,13 +83,9 @@ class Scheduler(object):
request=request, reason=e)
return
else:
if self.stats:
self.stats.inc_value('scheduler/disk_enqueued', spider=self.spider)
return True
def _mqpush(self, request):
if self.stats:
self.stats.inc_value('scheduler/memory_enqueued', spider=self.spider)
self.mqs.push(request, -request.priority)
def _dqpop(self):