From b037ef6a27d01b6e29ebaaf9b53030511a64eae7 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 22 Jun 2009 18:37:39 -0300 Subject: [PATCH] added clear_pending_requests to scheduler --- scrapy/core/scheduler/schedulers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapy/core/scheduler/schedulers.py b/scrapy/core/scheduler/schedulers.py index 95a2f605f..22af11b02 100644 --- a/scrapy/core/scheduler/schedulers.py +++ b/scrapy/core/scheduler/schedulers.py @@ -3,8 +3,10 @@ The Scrapy Scheduler """ from twisted.internet import defer +from twisted.python.failure import Failure from scrapy.utils.datatypes import PriorityQueue, PriorityStack +from scrapy.core.exceptions import IgnoreRequest from scrapy.conf import settings from .middleware import SchedulerMiddlewareManager @@ -57,6 +59,13 @@ class Scheduler(object): self.pending_requests[domain].push((request, dfd), -request.priority) return dfd + def clear_pending_requests(self, domain): + """Remove all pending requests for the given domain""" + q = self.pending_requests[domain] + while q: + _, dfd = q.pop()[0] + dfd.errback(Failure(IgnoreRequest())) + def next_request(self, domain): """Return the next available request to be downloaded for a domain.