diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py index 645393d6b..0a3d54596 100644 --- a/scrapy/contrib/closespider.py +++ b/scrapy/contrib/closespider.py @@ -11,27 +11,28 @@ from twisted.python import log as txlog from scrapy import signals, log + class CloseSpider(object): def __init__(self, crawler): self.crawler = crawler - self.timeout = crawler.settings.getfloat('CLOSESPIDER_TIMEOUT') - self.itemcount = crawler.settings.getint('CLOSESPIDER_ITEMCOUNT') - self.pagecount = crawler.settings.getint('CLOSESPIDER_PAGECOUNT') - self.errorcount = crawler.settings.getint('CLOSESPIDER_ERRORCOUNT') - self.errorcounts = defaultdict(int) - self.pagecounts = defaultdict(int) - self.counts = defaultdict(int) - self.tasks = {} + self.close_on = { + 'timeout': crawler.settings.getfloat('CLOSESPIDER_TIMEOUT'), + 'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'), + 'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'), + 'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'), + } - if self.errorcount: + self.counter = defaultdict(int) + + if self.close_on.get('errorcount'): txlog.addObserver(self.catch_log) - if self.pagecount: + if self.close_on.get('pagecount'): crawler.signals.connect(self.page_count, signal=signals.response_received) - if self.timeout: + if self.close_on.get('timeout'): crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) - if self.itemcount: + if self.close_on.get('itemcount'): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @@ -43,29 +44,26 @@ class CloseSpider(object): if event.get('logLevel') == log.ERROR: spider = event.get('spider') if spider: - self.errorcounts[spider] += 1 - if self.errorcounts[spider] == self.errorcount: + self.counter['errorcount'] += 1 + if self.counter['errorcount'] == self.close_on['errorcount']: self.crawler.engine.close_spider(spider, 'closespider_errorcount') def page_count(self, response, request, spider): - self.pagecounts[spider] += 1 - if self.pagecounts[spider] == self.pagecount: + self.counter['pagecount'] += 1 + if self.counter['pagecount'] == self.close_on['pagecount']: self.crawler.engine.close_spider(spider, 'closespider_pagecount') def spider_opened(self, spider): - self.tasks[spider] = reactor.callLater(self.timeout, \ - self.crawler.engine.close_spider, spider=spider, \ + self.task = reactor.callLater(self.close_on['timeout'], \ + self.crawler.engine.close_spider, spider, \ reason='closespider_timeout') def item_scraped(self, item, spider): - self.counts[spider] += 1 - if self.counts[spider] == self.itemcount: + self.counter['itemcount'] += 1 + if self.counter['itemcount'] == self.close_on['itemcount']: self.crawler.engine.close_spider(spider, 'closespider_itemcount') def spider_closed(self, spider): - self.counts.pop(spider, None) - self.pagecounts.pop(spider, None) - self.errorcounts.pop(spider, None) - tsk = self.tasks.pop(spider, None) - if tsk and tsk.active(): - tsk.cancel() + task = getattr(self, 'task', False) + if task and task.active(): + task.cancel() diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py index f5c12220d..27442d25b 100644 --- a/scrapy/tests/spiders.py +++ b/scrapy/tests/spiders.py @@ -6,14 +6,29 @@ import time from scrapy.spider import BaseSpider from scrapy.http import Request +from scrapy.item import Item from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor -class FollowAllSpider(BaseSpider): + +class MetaSpider(BaseSpider): + + name = 'meta' + + def __init__(self, *args, **kwargs): + super(MetaSpider, self).__init__(*args, **kwargs) + self.meta = {} + + def closed(self, reason): + self.meta['close_reason'] = reason + + +class FollowAllSpider(MetaSpider): name = 'follow' link_extractor = SgmlLinkExtractor() - def __init__(self, total=10, show=20, order="rand"): + def __init__(self, total=10, show=20, order="rand", *args, **kwargs): + super(FollowAllSpider, self).__init__(*args, **kwargs) self.urls_visited = [] self.times = [] url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order) @@ -25,11 +40,13 @@ class FollowAllSpider(BaseSpider): for link in self.link_extractor.extract_links(response): yield Request(link.url, callback=self.parse) -class DelaySpider(BaseSpider): + +class DelaySpider(MetaSpider): name = 'delay' - def __init__(self, n=1): + def __init__(self, n=1, *args, **kwargs): + super(DelaySpider, self).__init__(*args, **kwargs) self.n = n self.t1 = self.t2 = self.t2_err = 0 @@ -44,12 +61,42 @@ class DelaySpider(BaseSpider): def errback(self, failure): self.t2_err = time.time() -class SimpleSpider(BaseSpider): + +class SimpleSpider(MetaSpider): name = 'simple' - def __init__(self, url="http://localhost:8998"): + def __init__(self, url="http://localhost:8998", *args, **kwargs): + super(SimpleSpider, self).__init__(*args, **kwargs) self.start_urls = [url] def parse(self, response): self.log("Got response %d" % response.status) + + +class ItemSpider(FollowAllSpider): + + name = 'item' + + def parse(self, response): + for request in super(ItemSpider, self).parse(response): + yield request + yield Item() + + +class DefaultError(Exception): + pass + + +class ErrorSpider(FollowAllSpider): + + name = 'error' + exception_cls = DefaultError + + def raise_exception(self): + raise self.exception_cls('Expected exception') + + def parse(self, response): + for request in super(ErrorSpider, self).parse(response): + yield request + self.raise_exception() diff --git a/scrapy/tests/test_closespider.py b/scrapy/tests/test_closespider.py new file mode 100644 index 000000000..290067f03 --- /dev/null +++ b/scrapy/tests/test_closespider.py @@ -0,0 +1,67 @@ +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from scrapy.utils.test import get_crawler +from scrapy.tests.spiders import FollowAllSpider, ItemSpider, ErrorSpider +from scrapy.tests.mockserver import MockServer + + +def docrawl(spider, settings=None): + crawler = get_crawler(settings) + crawler.configure() + crawler.crawl(spider) + return crawler.start() + +class TestCloseSpider(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_closespider_itemcount(self): + spider = ItemSpider() + close_on = 5 + yield docrawl(spider, {'CLOSESPIDER_ITEMCOUNT': close_on}) + reason = spider.meta['close_reason'] + self.assertEqual(reason, 'closespider_itemcount') + itemcount = spider.crawler.stats.get_value('item_scraped_count') + self.assertTrue(itemcount >= close_on) + + @defer.inlineCallbacks + def test_closespider_pagecount(self): + spider = FollowAllSpider() + close_on = 5 + yield docrawl(spider, {'CLOSESPIDER_PAGECOUNT': close_on}) + reason = spider.meta['close_reason'] + self.assertEqual(reason, 'closespider_pagecount') + pagecount = spider.crawler.stats.get_value('response_received_count') + self.assertTrue(pagecount >= close_on) + + @defer.inlineCallbacks + def test_closespider_errorcount(self): + spider = ErrorSpider(total=1000000) + close_on = 5 + yield docrawl(spider, {'CLOSESPIDER_ERRORCOUNT': close_on}) + self.flushLoggedErrors(spider.exception_cls) + reason = spider.meta['close_reason'] + self.assertEqual(reason, 'closespider_errorcount') + key = 'spider_exceptions/{name}'\ + .format(name=spider.exception_cls.__name__) + errorcount = spider.crawler.stats.get_value(key) + self.assertTrue(errorcount >= close_on) + + @defer.inlineCallbacks + def test_closespider_timeout(self): + spider = FollowAllSpider(total=1000000) + close_on = 0.1 + yield docrawl(spider, {'CLOSESPIDER_TIMEOUT': close_on}) + reason = spider.meta['close_reason'] + self.assertEqual(reason, 'closespider_timeout') + stats = spider.crawler.stats + start = stats.get_value('start_time') + stop = stats.get_value('finish_time') + diff = stop - start + self.assertTrue(diff.total_seconds() >= close_on)