From 031eb1e5edefe1c5841e110f6e50d24e8663ec62 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 28 May 2010 17:27:15 -0300 Subject: [PATCH] removed no longer used SpiderScheduler (obsoleted by ExecutionQueue) --- docs/topics/settings.rst | 10 --------- scrapy/conf/default_settings.py | 2 -- scrapy/contrib/spiderscheduler.py | 37 ------------------------------- 3 files changed, 49 deletions(-) delete mode 100644 scrapy/contrib/spiderscheduler.py diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8c06f0bd9..e40e0ac13 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -960,16 +960,6 @@ Example:: SPIDER_MODULES = ['mybot.spiders_prod', 'mybot.spiders_dev'] -.. setting:: SPIDER_SCHEDULER - -SPIDER_SCHEDULER ----------------- - -Default: ``'scrapy.contrib.spiderscheduler.FifoSpiderScheduler'`` - -The Spider Scheduler to use. The spider scheduler returns the next spider to -scrape. - .. setting:: STATS_CLASS STATS_CLASS diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 1bc2c8603..635e549ea 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -208,8 +208,6 @@ SPIDER_MIDDLEWARES_BASE = { SPIDER_MODULES = [] -SPIDER_SCHEDULER = 'scrapy.contrib.spiderscheduler.FifoSpiderScheduler' - SPIDERPROFILER_ENABLED = False STATS_CLASS = 'scrapy.stats.collector.MemoryStatsCollector' diff --git a/scrapy/contrib/spiderscheduler.py b/scrapy/contrib/spiderscheduler.py deleted file mode 100644 index e2c22a773..000000000 --- a/scrapy/contrib/spiderscheduler.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -The Spider Scheduler keeps track of next spiders to scrape. They must implement -the following methods: - -* next_spider() - return next spider to scrape and remove it from pending queue - -* add_spider(spider) - add spider to pending queue - -* remove_pending_spider(spider) - remove (all occurrences) of spider from pending queue, do nothing if not - pending - -* has_pending_spider(spider) - Return ``True`` if the spider is pending to scrape, ``False`` otherwise - -""" - -class FifoSpiderScheduler(object): - """Basic spider scheduler based on a FIFO queue""" - - def __init__(self): - self._pending_spiders = [] - - def next_spider(self) : - if self._pending_spiders: - return self._pending_spiders.pop(0) - - def add_spider(self, spider): - self._pending_spiders.append(spider) - - def remove_pending_spider(self, spider): - self._pending_spiders = [d for d in self._pending_spiders if d != spider] - - def has_pending_spider(self, spider): - return spider in self._pending_spiders