diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py index 91ae78c62..f96367905 100644 --- a/scrapy/contrib/spiders/sitemap.py +++ b/scrapy/contrib/spiders/sitemap.py @@ -22,7 +22,7 @@ class SitemapSpider(BaseSpider): self._follow = [regex(x) for x in self.sitemap_follow] def start_requests(self): - return [Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls] + return (Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls) def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): diff --git a/scrapy/spider.py b/scrapy/spider.py index cb090e3db..1829a9d3b 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -50,10 +50,8 @@ class BaseSpider(object_ref): return self._settings def start_requests(self): - reqs = [] for url in self.start_urls: - reqs.extend(arg_to_iter(self.make_requests_from_url(url))) - return reqs + yield self.make_requests_from_url(url) def make_requests_from_url(self, url): return Request(url, dont_filter=True)