diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index b07853568..8587161b2 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -8,6 +8,7 @@ See documentation in docs/topics/spiders.rst import copy from scrapy.http import Request +from scrapy.utils.spider import iterate_spider_output from scrapy.contrib.spiders.init import InitSpider from scrapy.conf import settings @@ -116,7 +117,7 @@ class CrawlSpider(InitSpider): if callback: cb_res = callback(response, **cb_kwargs) or () cb_res = self.process_results(response, cb_res) - res.extend(cb_res) + res.extend(iterate_spider_output(cb_res)) return res def _compile_rules(self): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index aa2473787..b20659cf1 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -5,7 +5,8 @@ from twisted.python.failure import Failure from twisted.internet import defer from scrapy.utils.defer import defer_result, defer_succeed, parallel -from scrapy.utils.misc import load_object, arg_to_iter +from scrapy.utils.spider import iterate_spider_output +from scrapy.utils.misc import load_object from scrapy.utils.signal import send_catch_log from scrapy.core.exceptions import IgnoreRequest, DropItem from scrapy.core import signals @@ -16,6 +17,7 @@ from scrapy import log from scrapy.stats import stats from scrapy.conf import settings + class SpiderInfo(object): """Object for holding data of the responses being scraped""" @@ -128,10 +130,7 @@ class Scraper(object): def call_spider(self, result, request, spider): defer_result(result).chainDeferred(request.deferred) - return request.deferred.addCallback(self._iterable_spider_output) - - def _iterable_spider_output(self, result): - return [result] if isinstance(result, BaseItem) else arg_to_iter(result) + return request.deferred.addCallback(iterate_spider_output) def handle_spider_error(self, _failure, request, spider, propagated_failure=None): referer = request.headers.get('Referer', None) diff --git a/scrapy/tests/test_utils_spider.py b/scrapy/tests/test_utils_spider.py new file mode 100644 index 000000000..9966f136b --- /dev/null +++ b/scrapy/tests/test_utils_spider.py @@ -0,0 +1,22 @@ +import unittest +from scrapy.http import Request +from scrapy.item import BaseItem +from scrapy.utils.spider import iterate_spider_output + + +class UtilsSpidersTestCase(unittest.TestCase): + + def test_iterate_spider_output(self): + i = BaseItem() + r = Request('http://scrapytest.org') + o = object() + + self.assertEqual(list(iterate_spider_output(i)), [i]) + self.assertEqual(list(iterate_spider_output(r)), [r]) + self.assertEqual(list(iterate_spider_output(o)), [o]) + self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o]) + + +if __name__ == "__main__": + unittest.main() + diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py new file mode 100644 index 000000000..9307fc234 --- /dev/null +++ b/scrapy/utils/spider.py @@ -0,0 +1,7 @@ +from scrapy.item import BaseItem +from scrapy.utils.misc import arg_to_iter + + +def iterate_spider_output(result): + return [result] if isinstance(result, BaseItem) else arg_to_iter(result) +