Merge pull request #313 from nramirezuy/closespider-6017

drop multi-spider support from CloseSpider extension

scrapy/contrib/closespider.py

@@ -11,27 +11,28 @@ from twisted.python import log as txlog
 
 from scrapy import signals, log
 
+
 class CloseSpider(object):
 
     def __init__(self, crawler):
         self.crawler = crawler
-        self.timeout = crawler.settings.getfloat('CLOSESPIDER_TIMEOUT')
-        self.itemcount = crawler.settings.getint('CLOSESPIDER_ITEMCOUNT')
-        self.pagecount = crawler.settings.getint('CLOSESPIDER_PAGECOUNT')
-        self.errorcount = crawler.settings.getint('CLOSESPIDER_ERRORCOUNT')
 
-        self.errorcounts = defaultdict(int)
-        self.pagecounts = defaultdict(int)
-        self.counts = defaultdict(int)
-        self.tasks = {}
+        self.close_on = {
+            'timeout': crawler.settings.getfloat('CLOSESPIDER_TIMEOUT'),
+            'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'),
+            'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'),
+            'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
+            }
 
-        if self.errorcount:
+        self.counter = defaultdict(int)
+
+        if self.close_on.get('errorcount'):
             txlog.addObserver(self.catch_log)
-        if self.pagecount:
+        if self.close_on.get('pagecount'):
             crawler.signals.connect(self.page_count, signal=signals.response_received)
-        if self.timeout:
+        if self.close_on.get('timeout'):
             crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)
-        if self.itemcount:
+        if self.close_on.get('itemcount'):
             crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
         crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
 
@@ -43,29 +44,26 @@ class CloseSpider(object):
         if event.get('logLevel') == log.ERROR:
             spider = event.get('spider')
             if spider:
-                self.errorcounts[spider] += 1
-                if self.errorcounts[spider] == self.errorcount:
+                self.counter['errorcount'] += 1
+                if self.counter['errorcount'] == self.close_on['errorcount']:
                     self.crawler.engine.close_spider(spider, 'closespider_errorcount')
 
     def page_count(self, response, request, spider):
-        self.pagecounts[spider] += 1
-        if self.pagecounts[spider] == self.pagecount:
+        self.counter['pagecount'] += 1
+        if self.counter['pagecount'] == self.close_on['pagecount']:
             self.crawler.engine.close_spider(spider, 'closespider_pagecount')
 
     def spider_opened(self, spider):
-        self.tasks[spider] = reactor.callLater(self.timeout, \
-            self.crawler.engine.close_spider, spider=spider, \
+        self.task = reactor.callLater(self.close_on['timeout'], \
+            self.crawler.engine.close_spider, spider, \
             reason='closespider_timeout')
 
     def item_scraped(self, item, spider):
-        self.counts[spider] += 1
-        if self.counts[spider] == self.itemcount:
+        self.counter['itemcount'] += 1
+        if self.counter['itemcount'] == self.close_on['itemcount']:
             self.crawler.engine.close_spider(spider, 'closespider_itemcount')
 
     def spider_closed(self, spider):
-        self.counts.pop(spider, None)
-        self.pagecounts.pop(spider, None)
-        self.errorcounts.pop(spider, None)
-        tsk = self.tasks.pop(spider, None)
-        if tsk and tsk.active():
-            tsk.cancel()
+        task = getattr(self, 'task', False)
+        if task and task.active():
+            task.cancel()

scrapy/tests/spiders.py

@@ -6,14 +6,29 @@ import time
 
 from scrapy.spider import BaseSpider
 from scrapy.http import Request
+from scrapy.item import Item
 from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
 
-class FollowAllSpider(BaseSpider):
+
+class MetaSpider(BaseSpider):
+
+    name = 'meta'
+
+    def __init__(self, *args, **kwargs):
+        super(MetaSpider, self).__init__(*args, **kwargs)
+        self.meta = {}
+
+    def closed(self, reason):
+        self.meta['close_reason'] = reason
+
+
+class FollowAllSpider(MetaSpider):
 
     name = 'follow'
     link_extractor = SgmlLinkExtractor()
 
-    def __init__(self, total=10, show=20, order="rand"):
+    def __init__(self, total=10, show=20, order="rand", *args, **kwargs):
+        super(FollowAllSpider, self).__init__(*args, **kwargs)
         self.urls_visited = []
         self.times = []
         url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order)
@@ -25,11 +40,13 @@ class FollowAllSpider(BaseSpider):
         for link in self.link_extractor.extract_links(response):
             yield Request(link.url, callback=self.parse)
 
-class DelaySpider(BaseSpider):
+
+class DelaySpider(MetaSpider):
 
     name = 'delay'
 
-    def __init__(self, n=1):
+    def __init__(self, n=1, *args, **kwargs):
+        super(DelaySpider, self).__init__(*args, **kwargs)
         self.n = n
         self.t1 = self.t2 = self.t2_err = 0
 
@@ -44,12 +61,42 @@ class DelaySpider(BaseSpider):
     def errback(self, failure):
         self.t2_err = time.time()
 
-class SimpleSpider(BaseSpider):
+
+class SimpleSpider(MetaSpider):
 
     name = 'simple'
 
-    def __init__(self, url="http://localhost:8998"):
+    def __init__(self, url="http://localhost:8998", *args, **kwargs):
+        super(SimpleSpider, self).__init__(*args, **kwargs)
         self.start_urls = [url]
 
     def parse(self, response):
         self.log("Got response %d" % response.status)
+
+
+class ItemSpider(FollowAllSpider):
+
+    name = 'item'
+
+    def parse(self, response):
+        for request in super(ItemSpider, self).parse(response):
+            yield request
+            yield Item()
+
+
+class DefaultError(Exception):
+    pass
+
+
+class ErrorSpider(FollowAllSpider):
+
+    name = 'error'
+    exception_cls = DefaultError
+
+    def raise_exception(self):
+        raise self.exception_cls('Expected exception')
+
+    def parse(self, response):
+        for request in super(ErrorSpider, self).parse(response):
+            yield request
+            self.raise_exception()

scrapy/tests/test_closespider.py

@@ -0,0 +1,67 @@
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+from scrapy.utils.test import get_crawler
+from scrapy.tests.spiders import FollowAllSpider, ItemSpider, ErrorSpider
+from scrapy.tests.mockserver import MockServer
+
+
+def docrawl(spider, settings=None):
+    crawler = get_crawler(settings)
+    crawler.configure()
+    crawler.crawl(spider)
+    return crawler.start()
+
+class TestCloseSpider(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_closespider_itemcount(self):
+        spider = ItemSpider()
+        close_on = 5
+        yield docrawl(spider, {'CLOSESPIDER_ITEMCOUNT': close_on})
+        reason = spider.meta['close_reason']
+        self.assertEqual(reason, 'closespider_itemcount')
+        itemcount = spider.crawler.stats.get_value('item_scraped_count')
+        self.assertTrue(itemcount >= close_on)
+
+    @defer.inlineCallbacks
+    def test_closespider_pagecount(self):
+        spider = FollowAllSpider()
+        close_on = 5
+        yield docrawl(spider, {'CLOSESPIDER_PAGECOUNT': close_on})
+        reason = spider.meta['close_reason']
+        self.assertEqual(reason, 'closespider_pagecount')
+        pagecount = spider.crawler.stats.get_value('response_received_count')
+        self.assertTrue(pagecount >= close_on)
+
+    @defer.inlineCallbacks
+    def test_closespider_errorcount(self):
+        spider = ErrorSpider(total=1000000)
+        close_on = 5
+        yield docrawl(spider, {'CLOSESPIDER_ERRORCOUNT': close_on})
+        self.flushLoggedErrors(spider.exception_cls)
+        reason = spider.meta['close_reason']
+        self.assertEqual(reason, 'closespider_errorcount')
+        key = 'spider_exceptions/{name}'\
+                .format(name=spider.exception_cls.__name__)
+        errorcount = spider.crawler.stats.get_value(key)
+        self.assertTrue(errorcount >= close_on)
+
+    @defer.inlineCallbacks
+    def test_closespider_timeout(self):
+        spider = FollowAllSpider(total=1000000)
+        close_on = 0.1
+        yield docrawl(spider, {'CLOSESPIDER_TIMEOUT': close_on})
+        reason = spider.meta['close_reason']
+        self.assertEqual(reason, 'closespider_timeout')
+        stats = spider.crawler.stats
+        start = stats.get_value('start_time')
+        stop = stats.get_value('finish_time')
+        diff = stop - start
+        self.assertTrue(diff.total_seconds() >= close_on)