From bc41d313771ad0047b1caacd2d544d960ed8dcda Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 23 Apr 2018 10:45:14 +0000 Subject: [PATCH] new method with ability to control consumption --- scrapy/spiders/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e9c131e3b..a522c6af9 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -82,6 +82,13 @@ class Spider(object_ref): for url in self.start_urls: yield Request(url, dont_filter=True) + def start_requests_with_control(self): + for r in self.start_requests(): + yield r + + if r is not None: + yield None + def make_requests_from_url(self, url): """ This method is deprecated. """ return Request(url, dont_filter=True)