From 88dbe2ae8785ab96ce0553ed8899c021fb9ac109 Mon Sep 17 00:00:00 2001 From: Shane Evans Date: Fri, 20 May 2011 14:35:37 +0100 Subject: [PATCH] fix error messages due to fetching pages during shutdown process This version keeps the faster approach of not processing request callbacks when engine is shutting down --- scrapy/core/scheduler.py | 2 +- scrapy/utils/datatypes.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 2b167a6dc..0a2a15b48 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -58,7 +58,7 @@ class Scheduler(object): return dfd def clear_pending_requests(self, spider): - pass + self.pending_requests[spider].clear() def next_request(self, spider): """Return the next available request to be downloaded for a spider. diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 044bcae27..621934657 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -305,6 +305,11 @@ class PriorityQueue(object): del self.positems[priority] return t raise IndexError("pop from an empty queue") + + def clear(self): + self.negitems.clear() + self.pzero.clear() + self.positems.clear() def __len__(self): total = sum(len(v) for v in self.negitems.values()) + \