From 18301b7e66c6ab837dd24d6f14aa25a84a432019 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 25 Jun 2009 14:13:45 -0300 Subject: [PATCH] downloader: performance improvement for sites that use download delay (replace datetime by time) --- scrapy/core/downloader/manager.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py index 380288928..0890d7ca4 100644 --- a/scrapy/core/downloader/manager.py +++ b/scrapy/core/downloader/manager.py @@ -2,7 +2,7 @@ Download web pages using asynchronous IO """ -import datetime +from time import time from twisted.internet import reactor, defer @@ -35,7 +35,7 @@ class SiteInfo(object): self.active = set() self.transferring = set() self.closing = False - self.lastseen = None + self.lastseen = 0 def free_transfer_slots(self): return self.max_concurrent_requests - len(self.transferring) @@ -97,10 +97,9 @@ class Downloader(object): return # Delay queue processing if a download_delay is configured - now = datetime.datetime.now() - if site.download_delay and site.lastseen: - delta = now - site.lastseen - penalty = site.download_delay - delta.seconds + now = time() + if site.download_delay: + penalty = site.download_delay - now + site.lastseen if penalty > 0: reactor.callLater(penalty, self.process_queue, spider=spider) return