diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 5993ab68a..b68f976e5 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -279,10 +279,10 @@ report will also be sent to those addresses. Close domain extension ~~~~~~~~~~~~~~~~~~~~~~ -.. module:: scrapy.contrib.closedomain +.. module:: scrapy.contrib.closespider :synopsis: Close domain extension -.. class:: scrapy.contrib.closedomain.CloseDomain +.. class:: scrapy.contrib.closespider.CloseSpider Closes a domain/spider automatically when some conditions are met, using a specific closing reason for each condition. @@ -290,28 +290,28 @@ specific closing reason for each condition. The conditions for closing a domain can be configured through the following settings. Other conditions will be supported in the future. -.. setting:: CLOSEDOMAIN_TIMEOUT +.. setting:: CLOSESPIDER_TIMEOUT -CLOSEDOMAIN_TIMEOUT +CLOSESPIDER_TIMEOUT """"""""""""""""""" Default: ``0`` An integer which specifies a number of seconds. If the domain remains open for more than that number of second, it will be automatically closed with the -reason ``closedomain_timeout``. If zero (or non set) domains won't be closed by +reason ``closespider_timeout``. If zero (or non set) domains won't be closed by timeout. -.. setting:: CLOSEDOMAIN_ITEMPASSED +.. setting:: CLOSESPIDER_ITEMPASSED -CLOSEDOMAIN_ITEMPASSED +CLOSESPIDER_ITEMPASSED """""""""""""""""""""" Default: ``0`` An integer which specifies a number of items. If the spider scrapes more than that amount if items and those items are passed by the item pipeline, the -domain will be closed with the reason ``closedomain_itempassed``. If zero (or +domain will be closed with the reason ``closespider_itempassed``. If zero (or non set) domains won't be closed by number of passed items. StatsMailer extension diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 0fd87cf6d..ecd76540f 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -18,8 +18,8 @@ from os.path import join, abspath, dirname BOT_NAME = 'scrapybot' BOT_VERSION = '1.0' -CLOSEDOMAIN_TIMEOUT = 0 -CLOSEDOMAIN_ITEMPASSED = 0 +CLOSESPIDER_TIMEOUT = 0 +CLOSESPIDER_ITEMPASSED = 0 COMMANDS_MODULE = '' COMMANDS_SETTINGS_MODULE = '' @@ -86,7 +86,7 @@ EXTENSIONS_BASE = { 'scrapy.contrib.webconsole.stats.StatsDump': 0, 'scrapy.contrib.memusage.MemoryUsage': 0, 'scrapy.contrib.memdebug.MemoryDebugger': 0, - 'scrapy.contrib.closedomain.CloseDomain': 0, + 'scrapy.contrib.closespider.CloseSpider': 0, } GROUPSETTINGS_ENABLED = False diff --git a/scrapy/contrib/closedomain.py b/scrapy/contrib/closespider.py similarity index 77% rename from scrapy/contrib/closedomain.py rename to scrapy/contrib/closespider.py index 1a7169030..ccd111a03 100644 --- a/scrapy/contrib/closedomain.py +++ b/scrapy/contrib/closespider.py @@ -1,4 +1,4 @@ -"""CloseDomain is an extension that forces spiders to be closed after certain +"""CloseSpider is an extension that forces spiders to be closed after certain conditions are met. See documentation in docs/topics/extensions.rst @@ -13,11 +13,11 @@ from scrapy.core import signals from scrapy.core.engine import scrapyengine from scrapy.conf import settings -class CloseDomain(object): +class CloseSpider(object): def __init__(self): - self.timeout = settings.getint('CLOSEDOMAIN_TIMEOUT') - self.itempassed = settings.getint('CLOSEDOMAIN_ITEMPASSED') + self.timeout = settings.getint('CLOSESPIDER_TIMEOUT') + self.itempassed = settings.getint('CLOSESPIDER_ITEMPASSED') self.counts = defaultdict(int) self.tasks = {} @@ -30,12 +30,12 @@ class CloseDomain(object): def spider_opened(self, spider): self.tasks[spider] = reactor.callLater(self.timeout, scrapyengine.close_spider, \ - spider=spider, reason='closedomain_timeout') + spider=spider, reason='closespider_timeout') def item_passed(self, item, spider): self.counts[spider] += 1 if self.counts[spider] == self.itempassed: - scrapyengine.close_spider(spider, 'closedomain_itempassed') + scrapyengine.close_spider(spider, 'closespider_itempassed') def spider_closed(self, spider): self.counts.pop(spider, None)