From 71d7df9273dae5da8765ca4dd7e7ed4dacad5309 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 23 Jan 2013 16:28:41 -0200 Subject: [PATCH] simplify InitSpider implementation, fixing one bug (closes #228) and adding support generators return from start_requests() (which the previous version didn't) --- scrapy/contrib/spiders/init.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py index f759fd81f..8de12bb23 100644 --- a/scrapy/contrib/spiders/init.py +++ b/scrapy/contrib/spiders/init.py @@ -1,31 +1,18 @@ from scrapy.spider import BaseSpider +from scrapy.utils.spider import iterate_spider_output class InitSpider(BaseSpider): """Base Spider with initialization facilities""" - def __init__(self, *a, **kw): - super(InitSpider, self).__init__(*a, **kw) - self._postinit_reqs = [] - self._init_complete = False - self._init_started = False - - def make_requests_from_url(self, url): - req = super(InitSpider, self).make_requests_from_url(url) - if self._init_complete: - return req - self._postinit_reqs.append(req) - if not self._init_started: - self._init_started = True - return self.init_request() + def start_requests(self): + self._postinit_reqs = super(InitSpider, self).start_requests() + return iterate_spider_output(self.init_request()) def initialized(self, response=None): """This method must be set as the callback of your last initialization request. See self.init_request() docstring for more info. """ - self._init_complete = True - reqs = self._postinit_reqs[:] - del self._postinit_reqs - return reqs + return self.__dict__.pop('_postinit_reqs') def init_request(self): """This function should return one initialization request, with the