diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 0c7eefc3e..e2a08aa5d 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -15,6 +15,9 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import method_is_overridden +def __flatten(list_): + return (item for sublist in list_ for item in sublist) + class Spider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this class. @@ -83,11 +86,8 @@ class Spider(object_ref): yield Request(url, dont_filter=True) def start_requests_with_control(self): - for r in self.start_requests(): - yield r - - if r != signals.WaitUntilQueueEmpty: - yield signals.WaitUntilQueueEmpty + gen_ = ((r, signals.WaitUntilQueueEmpty) if r != signals.WaitUntilQueueEmpty else (r,) for r in self.start_requests()) + return __flatten(gen_) def make_requests_from_url(self, url): """ This method is deprecated. """ diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 14d6067dd..d5babdded 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -240,7 +240,7 @@ with multiples lines pass class FaultySpider(SimpleSpider): - def start_requests_with_control(self): + def start_requests(self): raise TestError crawler = self.runner.create_crawler(FaultySpider)