diff --git a/scrapy/contrib/domainsch.py b/scrapy/contrib/domainsch.py index 0755cf23d..c0ffbdcba 100644 --- a/scrapy/contrib/domainsch.py +++ b/scrapy/contrib/domainsch.py @@ -1,18 +1,19 @@ """ -Domain Schedulers keep track of next domains to scrape. They must implement the -following methods: +The Domain Scheduler keeps track of next domains to scrape. They must implement +the following methods: * next_domain() return next domain to scrape and remove it from pending queue * add_domain(domain) - add domain to pending domains to scrape + add domain to pending queue * remove_pending_domain(domain) - remove domain from pendings, do nothing if not pending + remove (all occurrences) of domain from pending queue, do nothing if not + pending * has_pending_domain(domain) - Return ``True`` if the domain is pending, ``False`` otherwise + Return ``True`` if the domain is pending to scrape, ``False`` otherwise """ @@ -30,7 +31,7 @@ class FifoDomainScheduler(object): self.pending_domains.append(domain) def remove_pending_domain(self, domain): - self.pending_domains.remove(domain) + self.pending_domains = [d for d in self.pending_domains if d != domain] def has_pending_domain(self, domain): return domain in self.pending_domains