From 80db8abee28c84ead88d041175398be2e9449f50 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 6 Aug 2009 11:56:32 -0300 Subject: [PATCH] use time.time() instead of datetime in SpiderProfiler extensions, which is faster and simpler --- scrapy/contrib/spider/profiler.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scrapy/contrib/spider/profiler.py b/scrapy/contrib/spider/profiler.py index 25c47598b..0f2d9bb9e 100644 --- a/scrapy/contrib/spider/profiler.py +++ b/scrapy/contrib/spider/profiler.py @@ -6,10 +6,10 @@ caused by spiders code. The results are collected using the StatsCollector. This extension introduces a big impact on crawling performance, so enable only -when needed. +for debugging. """ -import datetime +from time import time from scrapy.xlib.pydispatch import dispatcher @@ -37,15 +37,14 @@ class SpiderProfiler(object): def _profiled_callback(self, function, spider): def new_callback(*args, **kwargs): - tbefore = datetime.datetime.now() + tbefore = time() mbefore = self._memusage() r = function(*args, **kwargs) - tafter = datetime.datetime.now() mafter = self._memusage() - ct = tafter-tbefore + ct = time() - tbefore domain = spider.domain_name - tcc = stats.get_value('profiling/total_callback_time', datetime.timedelta(0), domain=domain) - sct = stats.get_value('profiling/slowest_callback_time', datetime.timedelta(0), domain=domain) + tcc = stats.get_value('profiling/total_callback_time', 0, domain=domain) + sct = stats.get_value('profiling/slowest_callback_time', 0, domain=domain) stats.set_value('profiling/total_callback_time' % spider.domain_name, tcc+ct, domain=domain) if ct > sct: stats.set_value('profiling/slowest_callback_time', ct, domain=domain)