check iterable output of crawl spider and add tests for common function used to iterate spider output

scrapy/contrib/spiders/crawl.py

@@ -8,6 +8,7 @@ See documentation in docs/topics/spiders.rst
 import copy
 
 from scrapy.http import Request
+from scrapy.utils.spider import iterate_spider_output
 from scrapy.contrib.spiders.init import InitSpider
 from scrapy.conf import settings
 
@@ -116,7 +117,7 @@ class CrawlSpider(InitSpider):
         if callback:
             cb_res = callback(response, **cb_kwargs) or ()
             cb_res = self.process_results(response, cb_res)
-            res.extend(cb_res)
+            res.extend(iterate_spider_output(cb_res))
         return res
 
     def _compile_rules(self):

scrapy/core/scraper.py

@@ -5,7 +5,8 @@ from twisted.python.failure import Failure
 from twisted.internet import defer
 
 from scrapy.utils.defer import defer_result, defer_succeed, parallel
-from scrapy.utils.misc import load_object, arg_to_iter
+from scrapy.utils.spider import iterate_spider_output
+from scrapy.utils.misc import load_object
 from scrapy.utils.signal import send_catch_log
 from scrapy.core.exceptions import IgnoreRequest, DropItem
 from scrapy.core import signals
@@ -16,6 +17,7 @@ from scrapy import log
 from scrapy.stats import stats
 from scrapy.conf import settings
 
+
 class SpiderInfo(object):
     """Object for holding data of the responses being scraped"""
 
@@ -128,10 +130,7 @@ class Scraper(object):
 
     def call_spider(self, result, request, spider):
         defer_result(result).chainDeferred(request.deferred)
-        return request.deferred.addCallback(self._iterable_spider_output)
-
-    def _iterable_spider_output(self, result):
-        return [result] if isinstance(result, BaseItem) else arg_to_iter(result)
+        return request.deferred.addCallback(iterate_spider_output)
 
     def handle_spider_error(self, _failure, request, spider, propagated_failure=None):
         referer = request.headers.get('Referer', None)

scrapy/tests/test_utils_spider.py

@@ -0,0 +1,22 @@
+import unittest
+from scrapy.http import Request
+from scrapy.item import BaseItem
+from scrapy.utils.spider import iterate_spider_output
+
+
+class UtilsSpidersTestCase(unittest.TestCase):
+
+    def test_iterate_spider_output(self):
+        i = BaseItem()
+        r = Request('http://scrapytest.org')
+        o = object()
+
+        self.assertEqual(list(iterate_spider_output(i)), [i])
+        self.assertEqual(list(iterate_spider_output(r)), [r])
+        self.assertEqual(list(iterate_spider_output(o)), [o])
+        self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o])
+
+
+if __name__ == "__main__":
+    unittest.main()
+

scrapy/utils/spider.py

@@ -0,0 +1,7 @@
+from scrapy.item import BaseItem
+from scrapy.utils.misc import  arg_to_iter
+
+
+def iterate_spider_output(result):
+    return [result] if isinstance(result, BaseItem) else arg_to_iter(result)
+