RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use time.time() instead of datetime in SpiderProfiler extensions, which is faster and simpler

This commit is contained in:
Pablo Hoffman 2009-08-06 11:56:32 -03:00
parent de48406f10
commit 80db8abee2
1 changed files with 6 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scrapy/contrib/spider/profiler.py
View File

@ -6,10 +6,10 @@ caused by spiders code.
The results are collected using the StatsCollector.
This extension introduces a big impact on crawling performance, so enable only
when needed.
for debugging.
"""
import datetime
from time import time
from scrapy.xlib.pydispatch import dispatcher
@ -37,15 +37,14 @@ class SpiderProfiler(object):
def _profiled_callback(self, function, spider):
def new_callback(*args, **kwargs):
tbefore = datetime.datetime.now()
tbefore = time()
mbefore = self._memusage()
r = function(*args, **kwargs)
tafter = datetime.datetime.now()
mafter = self._memusage()
ct = tafter-tbefore
ct = time() - tbefore
domain = spider.domain_name
tcc = stats.get_value('profiling/total_callback_time', datetime.timedelta(0), domain=domain)
sct = stats.get_value('profiling/slowest_callback_time', datetime.timedelta(0), domain=domain)
tcc = stats.get_value('profiling/total_callback_time', 0, domain=domain)
sct = stats.get_value('profiling/slowest_callback_time', 0, domain=domain)
stats.set_value('profiling/total_callback_time' % spider.domain_name, tcc+ct, domain=domain)
if ct > sct:
stats.set_value('profiling/slowest_callback_time', ct, domain=domain)