From a66bef7925d2d9f33fecb7ac5e7ace499ea66cf9 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 5 Sep 2010 02:23:08 -0300 Subject: [PATCH] Make execution queue poll interval configurable through a new QUEUE_POLL_INTERVAL setting --- scrapy/conf/default_settings.py | 3 ++- scrapy/contrib/spiderqueue.py | 2 -- scrapy/crawler.py | 6 ++++-- scrapy/queue.py | 7 +++---- scrapy/tests/test_queue.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index d07d4e996..1d8502ece 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -178,6 +178,8 @@ MEMUSAGE_WARNING_MB = 0 NEWSPIDER_MODULE = '' +QUEUE_POLL_INTERVAL = 5 + RANDOMIZE_DOWNLOAD_DELAY = True REDIRECT_MAX_METAREFRESH_DELAY = 100 @@ -234,7 +236,6 @@ SQLITE_DB = 'scrapy.db' SQS_QUEUE = 'scrapy' SQS_VISIBILITY_TIMEOUT = 7200 -SQS_POLLING_DELAY = 30 SQS_REGION = 'us-east-1' STATS_CLASS = 'scrapy.statscol.MemoryStatsCollector' diff --git a/scrapy/contrib/spiderqueue.py b/scrapy/contrib/spiderqueue.py index 3f972b7d9..82d77402f 100644 --- a/scrapy/contrib/spiderqueue.py +++ b/scrapy/contrib/spiderqueue.py @@ -13,7 +13,6 @@ class SQSSpiderQueue(object): implements(ISpiderQueue) def __init__(self, *a, **kw): - self.polling_delay = kw.pop('polling_delay', 30) self.queue_name = kw.pop('queue_name', 'scrapy') self.region_name = kw.pop('region_name', 'us-east-1') self.visibility_timeout = kw.pop('visibility_timeout', 7200) @@ -26,7 +25,6 @@ class SQSSpiderQueue(object): @classmethod def from_settings(cls, settings): return cls( - polling_delay=settings.getint('SQS_POLLING_DELAY'), queue_name=settings['SQS_QUEUE'], region_name=settings['SQS_REGION'], visibility_timeout=settings.getint('SQS_VISIBILITY_TIMEOUT'), diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1d5d17366..7b153979e 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -37,7 +37,9 @@ class Crawler(object): spq_cls = load_object(self.settings['SPIDER_QUEUE_CLASS']) spq = spq_cls.from_settings(self.settings) keepalive = self.settings.getbool('KEEP_ALIVE') - self.queue = ExecutionQueue(self.spiders, spq, keepalive) + pollint = self.settings.getfloat('QUEUE_POLL_INTERVAL') + self.queue = ExecutionQueue(self.spiders, spq, poll_interval=pollint, + keep_alive=keepalive) self.engine = ExecutionEngine(self.settings, self._spider_closed) @defer.inlineCallbacks @@ -46,7 +48,7 @@ class Crawler(object): if spider: self._start_spider(spider, requests) if self.engine.has_capacity() and not self._nextcall.active(): - self._nextcall = reactor.callLater(self.queue.polling_delay, \ + self._nextcall = reactor.callLater(self.queue.poll_interval, \ self._spider_closed) @defer.inlineCallbacks diff --git a/scrapy/queue.py b/scrapy/queue.py index d23d43fa4..6868eaba6 100644 --- a/scrapy/queue.py +++ b/scrapy/queue.py @@ -8,13 +8,12 @@ from scrapy import log class ExecutionQueue(object): - polling_delay = 5 - - def __init__(self, spiders, queue, keepalive=False): + def __init__(self, spiders, queue, poll_interval=5, keep_alive=False): self.spider_requests = [] + self.poll_interval = poll_interval self._spiders = spiders self._queue = queue - self._keepalive = keepalive + self._keepalive = keep_alive def _append_next(self): """Called when there are no more items left in self.spider_requests. diff --git a/scrapy/tests/test_queue.py b/scrapy/tests/test_queue.py index 93e166492..f7a5e1bf2 100644 --- a/scrapy/tests/test_queue.py +++ b/scrapy/tests/test_queue.py @@ -30,7 +30,7 @@ class ExecutionQueueTest(unittest.TestCase): keep_alive = False def setUp(self): - self.queue = ExecutionQueue(TestSpiderManager(), None, self.keep_alive) + self.queue = ExecutionQueue(TestSpiderManager(), None, keep_alive=self.keep_alive) self.spider = TestSpider() self.request = Request('about:none')