From 728ec7c5c969288484eeb81dda0832d1240a2ad5 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 19 Jun 2009 19:41:56 -0300 Subject: [PATCH] minor adjustment to FifoDomainScheduler and improved documentation of domain scheduler API (remove_pending_domain method removes all ocurrences) --- scrapy/contrib/domainsch.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/contrib/domainsch.py b/scrapy/contrib/domainsch.py index 0755cf23d..c0ffbdcba 100644 --- a/scrapy/contrib/domainsch.py +++ b/scrapy/contrib/domainsch.py @@ -1,18 +1,19 @@ """ -Domain Schedulers keep track of next domains to scrape. They must implement the -following methods: +The Domain Scheduler keeps track of next domains to scrape. They must implement +the following methods: * next_domain() return next domain to scrape and remove it from pending queue * add_domain(domain) - add domain to pending domains to scrape + add domain to pending queue * remove_pending_domain(domain) - remove domain from pendings, do nothing if not pending + remove (all occurrences) of domain from pending queue, do nothing if not + pending * has_pending_domain(domain) - Return ``True`` if the domain is pending, ``False`` otherwise + Return ``True`` if the domain is pending to scrape, ``False`` otherwise """ @@ -30,7 +31,7 @@ class FifoDomainScheduler(object): self.pending_domains.append(domain) def remove_pending_domain(self, domain): - self.pending_domains.remove(domain) + self.pending_domains = [d for d in self.pending_domains if d != domain] def has_pending_domain(self, domain): return domain in self.pending_domains