From fd50891113cf44af7dca3a45f230ab8f744d4427 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 26 Nov 2009 17:00:16 -0200 Subject: [PATCH] Keep track of pending next-request calls in the engine and downloader, to cancel them properly when the spider is closed. This also avoids keeping spider references alive, after they are closed. --- scrapy/core/downloader/manager.py | 16 +++++++++++++--- scrapy/core/engine.py | 14 +++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py index 5498cc451..24bef99c4 100644 --- a/scrapy/core/downloader/manager.py +++ b/scrapy/core/downloader/manager.py @@ -36,6 +36,7 @@ class SiteInfo(object): self.transferring = set() self.closing = False self.lastseen = 0 + self.next_request_calls = set() def free_transfer_slots(self): return self.max_concurrent_requests - len(self.transferring) @@ -44,6 +45,10 @@ class SiteInfo(object): # use self.active to include requests in the downloader middleware return len(self.active) > 2 * self.max_concurrent_requests + def cancel_request_calls(self): + for call in self.next_request_calls: + call.cancel() + self.next_request_calls.clear() class Downloader(object): """Mantain many concurrent downloads and provide an HTTP abstraction. @@ -97,7 +102,11 @@ class Downloader(object): if site.download_delay: penalty = site.download_delay - now + site.lastseen if penalty > 0: - reactor.callLater(penalty, self._process_queue, spider=spider) + d = defer.Deferred() + d.addCallback(self._process_queue) + call = reactor.callLater(penalty, d.callback, spider) + site.next_request_calls.add(call) + d.addBoth(lambda x: site.next_request_calls.remove(call)) return site.lastseen = now @@ -153,12 +162,13 @@ class Downloader(object): def close_spider(self, spider): """Free any resources associated with the given spider""" - domain = spider.domain_name site = self.sites.get(spider) if not site or site.closing: - raise RuntimeError('Downloader spider already closed: %s' % domain) + raise RuntimeError('Downloader spider already closed: %s' % \ + spider.domain_name) site.closing = True + site.cancel_request_calls() self._process_queue(spider) def has_capacity(self): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 81d8c59ed..11cb0e98e 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -32,7 +32,7 @@ class ExecutionEngine(object): self.running = False self.killed = False self.paused = False - self._next_request_pending = set() + self._next_request_calls = {} self._mainloop_task = task.LoopingCall(self._mainloop) self._crawled_logline = load_object(settings['LOG_FORMATTER_CRAWLED']) @@ -107,10 +107,11 @@ class ExecutionEngine(object): The spider is closed if there are no more pages to scrape. """ if now: - self._next_request_pending.discard(spider) - elif spider not in self._next_request_pending: - self._next_request_pending.add(spider) - return reactor.callLater(0, self.next_request, spider, now=True) + self._next_request_calls.pop(spider, None) + elif spider not in self._next_request_calls: + call = reactor.callLater(0, self.next_request, spider, now=True) + self._next_request_calls[spider] = call + return call else: return @@ -307,6 +308,9 @@ class ExecutionEngine(object): send_catch_log(signal=signals.spider_closed, sender=self.__class__, \ spider=spider, reason=reason) stats.close_spider(spider, reason=reason) + call = self._next_request_calls.pop(spider, None) + if call and call.active(): + call.cancel() dfd = defer.maybeDeferred(spiders.close_spider, spider) dfd.addBoth(log.msg, "Spider closed (%s)" % reason, spider=spider) reactor.callLater(0, self._mainloop)