renamed CloseDomain extension to CloseSpider, and renamed CLOSEDOMAIN_* settings to CLOSESPIDER_*

--HG--
rename : scrapy/contrib/closedomain.py => scrapy/contrib/closespider.py
This commit is contained in:
Pablo Hoffman 2009-11-06 15:54:17 -02:00
parent cff4592b64
commit 74d0e82dbe
3 changed files with 17 additions and 17 deletions

View File

@ -279,10 +279,10 @@ report will also be sent to those addresses.
Close domain extension
~~~~~~~~~~~~~~~~~~~~~~
.. module:: scrapy.contrib.closedomain
.. module:: scrapy.contrib.closespider
:synopsis: Close domain extension
.. class:: scrapy.contrib.closedomain.CloseDomain
.. class:: scrapy.contrib.closespider.CloseSpider
Closes a domain/spider automatically when some conditions are met, using a
specific closing reason for each condition.
@ -290,28 +290,28 @@ specific closing reason for each condition.
The conditions for closing a domain can be configured through the following
settings. Other conditions will be supported in the future.
.. setting:: CLOSEDOMAIN_TIMEOUT
.. setting:: CLOSESPIDER_TIMEOUT
CLOSEDOMAIN_TIMEOUT
CLOSESPIDER_TIMEOUT
"""""""""""""""""""
Default: ``0``
An integer which specifies a number of seconds. If the domain remains open for
more than that number of second, it will be automatically closed with the
reason ``closedomain_timeout``. If zero (or non set) domains won't be closed by
reason ``closespider_timeout``. If zero (or non set) domains won't be closed by
timeout.
.. setting:: CLOSEDOMAIN_ITEMPASSED
.. setting:: CLOSESPIDER_ITEMPASSED
CLOSEDOMAIN_ITEMPASSED
CLOSESPIDER_ITEMPASSED
""""""""""""""""""""""
Default: ``0``
An integer which specifies a number of items. If the spider scrapes more than
that amount if items and those items are passed by the item pipeline, the
domain will be closed with the reason ``closedomain_itempassed``. If zero (or
domain will be closed with the reason ``closespider_itempassed``. If zero (or
non set) domains won't be closed by number of passed items.
StatsMailer extension

View File

@ -18,8 +18,8 @@ from os.path import join, abspath, dirname
BOT_NAME = 'scrapybot'
BOT_VERSION = '1.0'
CLOSEDOMAIN_TIMEOUT = 0
CLOSEDOMAIN_ITEMPASSED = 0
CLOSESPIDER_TIMEOUT = 0
CLOSESPIDER_ITEMPASSED = 0
COMMANDS_MODULE = ''
COMMANDS_SETTINGS_MODULE = ''
@ -86,7 +86,7 @@ EXTENSIONS_BASE = {
'scrapy.contrib.webconsole.stats.StatsDump': 0,
'scrapy.contrib.memusage.MemoryUsage': 0,
'scrapy.contrib.memdebug.MemoryDebugger': 0,
'scrapy.contrib.closedomain.CloseDomain': 0,
'scrapy.contrib.closespider.CloseSpider': 0,
}
GROUPSETTINGS_ENABLED = False

View File

@ -1,4 +1,4 @@
"""CloseDomain is an extension that forces spiders to be closed after certain
"""CloseSpider is an extension that forces spiders to be closed after certain
conditions are met.
See documentation in docs/topics/extensions.rst
@ -13,11 +13,11 @@ from scrapy.core import signals
from scrapy.core.engine import scrapyengine
from scrapy.conf import settings
class CloseDomain(object):
class CloseSpider(object):
def __init__(self):
self.timeout = settings.getint('CLOSEDOMAIN_TIMEOUT')
self.itempassed = settings.getint('CLOSEDOMAIN_ITEMPASSED')
self.timeout = settings.getint('CLOSESPIDER_TIMEOUT')
self.itempassed = settings.getint('CLOSESPIDER_ITEMPASSED')
self.counts = defaultdict(int)
self.tasks = {}
@ -30,12 +30,12 @@ class CloseDomain(object):
def spider_opened(self, spider):
self.tasks[spider] = reactor.callLater(self.timeout, scrapyengine.close_spider, \
spider=spider, reason='closedomain_timeout')
spider=spider, reason='closespider_timeout')
def item_passed(self, item, spider):
self.counts[spider] += 1
if self.counts[spider] == self.itempassed:
scrapyengine.close_spider(spider, 'closedomain_itempassed')
scrapyengine.close_spider(spider, 'closespider_itempassed')
def spider_closed(self, spider):
self.counts.pop(spider, None)