mirror of https://github.com/scrapy/scrapy.git
renamed CloseDomain extension to CloseSpider, and renamed CLOSEDOMAIN_* settings to CLOSESPIDER_*
--HG-- rename : scrapy/contrib/closedomain.py => scrapy/contrib/closespider.py
This commit is contained in:
parent
cff4592b64
commit
74d0e82dbe
|
|
@ -279,10 +279,10 @@ report will also be sent to those addresses.
|
|||
Close domain extension
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. module:: scrapy.contrib.closedomain
|
||||
.. module:: scrapy.contrib.closespider
|
||||
:synopsis: Close domain extension
|
||||
|
||||
.. class:: scrapy.contrib.closedomain.CloseDomain
|
||||
.. class:: scrapy.contrib.closespider.CloseSpider
|
||||
|
||||
Closes a domain/spider automatically when some conditions are met, using a
|
||||
specific closing reason for each condition.
|
||||
|
|
@ -290,28 +290,28 @@ specific closing reason for each condition.
|
|||
The conditions for closing a domain can be configured through the following
|
||||
settings. Other conditions will be supported in the future.
|
||||
|
||||
.. setting:: CLOSEDOMAIN_TIMEOUT
|
||||
.. setting:: CLOSESPIDER_TIMEOUT
|
||||
|
||||
CLOSEDOMAIN_TIMEOUT
|
||||
CLOSESPIDER_TIMEOUT
|
||||
"""""""""""""""""""
|
||||
|
||||
Default: ``0``
|
||||
|
||||
An integer which specifies a number of seconds. If the domain remains open for
|
||||
more than that number of second, it will be automatically closed with the
|
||||
reason ``closedomain_timeout``. If zero (or non set) domains won't be closed by
|
||||
reason ``closespider_timeout``. If zero (or non set) domains won't be closed by
|
||||
timeout.
|
||||
|
||||
.. setting:: CLOSEDOMAIN_ITEMPASSED
|
||||
.. setting:: CLOSESPIDER_ITEMPASSED
|
||||
|
||||
CLOSEDOMAIN_ITEMPASSED
|
||||
CLOSESPIDER_ITEMPASSED
|
||||
""""""""""""""""""""""
|
||||
|
||||
Default: ``0``
|
||||
|
||||
An integer which specifies a number of items. If the spider scrapes more than
|
||||
that amount if items and those items are passed by the item pipeline, the
|
||||
domain will be closed with the reason ``closedomain_itempassed``. If zero (or
|
||||
domain will be closed with the reason ``closespider_itempassed``. If zero (or
|
||||
non set) domains won't be closed by number of passed items.
|
||||
|
||||
StatsMailer extension
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ from os.path import join, abspath, dirname
|
|||
BOT_NAME = 'scrapybot'
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
CLOSEDOMAIN_TIMEOUT = 0
|
||||
CLOSEDOMAIN_ITEMPASSED = 0
|
||||
CLOSESPIDER_TIMEOUT = 0
|
||||
CLOSESPIDER_ITEMPASSED = 0
|
||||
|
||||
COMMANDS_MODULE = ''
|
||||
COMMANDS_SETTINGS_MODULE = ''
|
||||
|
|
@ -86,7 +86,7 @@ EXTENSIONS_BASE = {
|
|||
'scrapy.contrib.webconsole.stats.StatsDump': 0,
|
||||
'scrapy.contrib.memusage.MemoryUsage': 0,
|
||||
'scrapy.contrib.memdebug.MemoryDebugger': 0,
|
||||
'scrapy.contrib.closedomain.CloseDomain': 0,
|
||||
'scrapy.contrib.closespider.CloseSpider': 0,
|
||||
}
|
||||
|
||||
GROUPSETTINGS_ENABLED = False
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""CloseDomain is an extension that forces spiders to be closed after certain
|
||||
"""CloseSpider is an extension that forces spiders to be closed after certain
|
||||
conditions are met.
|
||||
|
||||
See documentation in docs/topics/extensions.rst
|
||||
|
|
@ -13,11 +13,11 @@ from scrapy.core import signals
|
|||
from scrapy.core.engine import scrapyengine
|
||||
from scrapy.conf import settings
|
||||
|
||||
class CloseDomain(object):
|
||||
class CloseSpider(object):
|
||||
|
||||
def __init__(self):
|
||||
self.timeout = settings.getint('CLOSEDOMAIN_TIMEOUT')
|
||||
self.itempassed = settings.getint('CLOSEDOMAIN_ITEMPASSED')
|
||||
self.timeout = settings.getint('CLOSESPIDER_TIMEOUT')
|
||||
self.itempassed = settings.getint('CLOSESPIDER_ITEMPASSED')
|
||||
|
||||
self.counts = defaultdict(int)
|
||||
self.tasks = {}
|
||||
|
|
@ -30,12 +30,12 @@ class CloseDomain(object):
|
|||
|
||||
def spider_opened(self, spider):
|
||||
self.tasks[spider] = reactor.callLater(self.timeout, scrapyengine.close_spider, \
|
||||
spider=spider, reason='closedomain_timeout')
|
||||
spider=spider, reason='closespider_timeout')
|
||||
|
||||
def item_passed(self, item, spider):
|
||||
self.counts[spider] += 1
|
||||
if self.counts[spider] == self.itempassed:
|
||||
scrapyengine.close_spider(spider, 'closedomain_itempassed')
|
||||
scrapyengine.close_spider(spider, 'closespider_itempassed')
|
||||
|
||||
def spider_closed(self, spider):
|
||||
self.counts.pop(spider, None)
|
||||
Loading…
Reference in New Issue