Make execution queue poll interval configurable through a new QUEUE_POLL_INTERVAL setting

scrapy/conf/default_settings.py

@@ -178,6 +178,8 @@ MEMUSAGE_WARNING_MB = 0
 
 NEWSPIDER_MODULE = ''
 
+QUEUE_POLL_INTERVAL = 5
+
 RANDOMIZE_DOWNLOAD_DELAY = True
 
 REDIRECT_MAX_METAREFRESH_DELAY = 100
@@ -234,7 +236,6 @@ SQLITE_DB = 'scrapy.db'
 
 SQS_QUEUE = 'scrapy'
 SQS_VISIBILITY_TIMEOUT = 7200
-SQS_POLLING_DELAY = 30
 SQS_REGION = 'us-east-1'
 
 STATS_CLASS = 'scrapy.statscol.MemoryStatsCollector'

scrapy/contrib/spiderqueue.py

@@ -13,7 +13,6 @@ class SQSSpiderQueue(object):
     implements(ISpiderQueue)
 
     def __init__(self, *a, **kw):
-        self.polling_delay = kw.pop('polling_delay', 30)
         self.queue_name = kw.pop('queue_name', 'scrapy')
         self.region_name = kw.pop('region_name', 'us-east-1')
         self.visibility_timeout = kw.pop('visibility_timeout', 7200)
@@ -26,7 +25,6 @@ class SQSSpiderQueue(object):
     @classmethod
     def from_settings(cls, settings):
         return cls(
-            polling_delay=settings.getint('SQS_POLLING_DELAY'),
             queue_name=settings['SQS_QUEUE'],
             region_name=settings['SQS_REGION'],
             visibility_timeout=settings.getint('SQS_VISIBILITY_TIMEOUT'),

scrapy/crawler.py

@@ -37,7 +37,9 @@ class Crawler(object):
         spq_cls = load_object(self.settings['SPIDER_QUEUE_CLASS'])
         spq = spq_cls.from_settings(self.settings)
         keepalive = self.settings.getbool('KEEP_ALIVE')
-        self.queue = ExecutionQueue(self.spiders, spq, keepalive)
+        pollint = self.settings.getfloat('QUEUE_POLL_INTERVAL')
+        self.queue = ExecutionQueue(self.spiders, spq, poll_interval=pollint,
+            keep_alive=keepalive)
         self.engine = ExecutionEngine(self.settings, self._spider_closed)
 
     @defer.inlineCallbacks
@@ -46,7 +48,7 @@ class Crawler(object):
         if spider:
             self._start_spider(spider, requests)
         if self.engine.has_capacity() and not self._nextcall.active():
-            self._nextcall = reactor.callLater(self.queue.polling_delay, \
+            self._nextcall = reactor.callLater(self.queue.poll_interval, \
                 self._spider_closed)
 
     @defer.inlineCallbacks

scrapy/queue.py

@@ -8,13 +8,12 @@ from scrapy import log
 
 class ExecutionQueue(object):
 
-    polling_delay = 5
-
-    def __init__(self, spiders, queue, keepalive=False):
+    def __init__(self, spiders, queue, poll_interval=5, keep_alive=False):
         self.spider_requests = []
+        self.poll_interval = poll_interval
         self._spiders = spiders
         self._queue = queue
-        self._keepalive = keepalive
+        self._keepalive = keep_alive
 
     def _append_next(self):
         """Called when there are no more items left in self.spider_requests.

scrapy/tests/test_queue.py

@@ -30,7 +30,7 @@ class ExecutionQueueTest(unittest.TestCase):
     keep_alive = False
 
     def setUp(self):
-        self.queue = ExecutionQueue(TestSpiderManager(), None, self.keep_alive)
+        self.queue = ExecutionQueue(TestSpiderManager(), None, keep_alive=self.keep_alive)
         self.spider = TestSpider()
         self.request = Request('about:none')