From 5b31dfe3c970f7e16dc911bb8b028a0fff54a7f9 Mon Sep 17 00:00:00 2001 From: terut Date: Mon, 13 Feb 2017 23:51:43 -0800 Subject: [PATCH] Separate building request from _requests_to_follow in CrawlSpider You just overwrite buiding request if you can use another request class because of something like splash-plugin. --- scrapy/spiders/crawl.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 031f649d6..e5ac72e18 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -48,6 +48,11 @@ class CrawlSpider(Spider): def process_results(self, response, results): return results + def _build_request(self, rule, link): + r = Request(url=link.url, callback=self._response_downloaded) + r.meta.update(rule=rule, link_text=link.text) + return r + def _requests_to_follow(self, response): if not isinstance(response, HtmlResponse): return @@ -59,8 +64,7 @@ class CrawlSpider(Spider): links = rule.process_links(links) for link in links: seen.add(link) - r = Request(url=link.url, callback=self._response_downloaded) - r.meta.update(rule=n, link_text=link.text) + r = self._build_request(n, link) yield rule.process_request(r) def _response_downloaded(self, response):