From 0bf9e4627cfcc477f23fb81edabbad85c24911bd Mon Sep 17 00:00:00 2001
From: Martin Santos <santos@insophia.com>
Date: Tue, 28 Sep 2010 16:29:37 -0300
Subject: [PATCH] added support to CloseSpider extension, for close the spider
 after N pages have been crawled. Using the CLOSESPIDER_PAGECOUNT setting.
 closes #253

---
 docs/topics/extensions.rst          | 14 ++++++++++++++
 scrapy/contrib/closespider.py       | 11 ++++++++++-
 scrapy/settings/default_settings.py |  1 +
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 539d4ef1f..e2f12fbce 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -302,6 +302,20 @@ that amount if items and those items are passed by the item pipeline, the
 spider will be closed with the reason ``closespider_itempassed``. If zero (or
 non set), spiders won't be closed by number of passed items.
 
+.. setting:: CLOSESPIDER_PAGECOUNT
+
+CLOSESPIDER_PAGECOUNT
+""""""""""""""""""""""
+
+Default: ``0``
+
+.. versionadded: 0.11
+
+An integer which specifies the maximum number of responses to crawl. If the spider
+crawls more than that, the spider will be closed with the reason
+``closespider_pagecount``. If zero (or non set), spiders won't be closed by
+number of crawled responses.
+
 StatsMailer extension
 ~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py
index a334c155a..0a04309cf 100644
--- a/scrapy/contrib/closespider.py
+++ b/scrapy/contrib/closespider.py
@@ -18,21 +18,30 @@ class CloseSpider(object):
     def __init__(self):
         self.timeout = settings.getint('CLOSESPIDER_TIMEOUT')
         self.itempassed = settings.getint('CLOSESPIDER_ITEMPASSED')
+        self.pagecount = settings.getint('CLOSESPIDER_PAGECOUNT')
 
+        self.pagecounts = defaultdict(int)
         self.counts = defaultdict(int)
         self.tasks = {}
 
+        if self.pagecount:
+            dispatcher.connect(self.page_count, signal=signals.response_received)
         if self.timeout:
             dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
         if self.itempassed:
             dispatcher.connect(self.item_passed, signal=signals.item_passed)
         dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
 
+    def page_count(self, response, request, spider):
+        self.pagecounts[spider] += 1
+        if self.pagecounts[spider] == self.pagecount:
+            crawler.engine.close_spider(spider, 'closespider_pagecount')
+
     def spider_opened(self, spider):
         self.tasks[spider] = reactor.callLater(self.timeout, \
             crawler.engine.close_spider, spider=spider, \
             reason='closespider_timeout')
-        
+
     def item_passed(self, item, spider):
         self.counts[spider] += 1
         if self.counts[spider] == self.itempassed:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 4f0071a96..0b6703189 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -19,6 +19,7 @@ BOT_NAME = 'scrapybot'
 BOT_VERSION = '1.0'
 
 CLOSESPIDER_TIMEOUT = 0
+CLOSESPIDER_PAGECOUNT = 0
 CLOSESPIDER_ITEMPASSED = 0
 
 COMMANDS_MODULE = ''