From cc7fc33833318bdafed640d3c1c6635bd3bacb88 Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Mon, 5 Mar 2012 12:49:17 -0200 Subject: [PATCH 1/2] change start_request to return a generator instead of a list, in order to allow to modify start_urls triggered by spider_opened signal --- scrapy/spider.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scrapy/spider.py b/scrapy/spider.py index cb090e3db..1829a9d3b 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -50,10 +50,8 @@ class BaseSpider(object_ref): return self._settings def start_requests(self): - reqs = [] for url in self.start_urls: - reqs.extend(arg_to_iter(self.make_requests_from_url(url))) - return reqs + yield self.make_requests_from_url(url) def make_requests_from_url(self, url): return Request(url, dont_filter=True) From f6179a927e553ca362365e23ed8b8878f773ba32 Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Mon, 5 Mar 2012 14:25:12 -0200 Subject: [PATCH 2/2] replace list by generator also in start_requests method of Sitemap spider --- scrapy/contrib/spiders/sitemap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py index 91ae78c62..f96367905 100644 --- a/scrapy/contrib/spiders/sitemap.py +++ b/scrapy/contrib/spiders/sitemap.py @@ -22,7 +22,7 @@ class SitemapSpider(BaseSpider): self._follow = [regex(x) for x in self.sitemap_follow] def start_requests(self): - return [Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls] + return (Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls) def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'):