RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Separate building request from _requests_to_follow in CrawlSpider 

You just overwrite buiding request if you can use another request class
because of something like splash-plugin.
This commit is contained in:
terut 2017-02-13 23:51:43 -08:00
parent 7dd7646e65
commit 5b31dfe3c9
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/spiders/crawl.py
View File

@ -48,6 +48,11 @@ class CrawlSpider(Spider):
def process_results(self, response, results):
return results
def _build_request(self, rule, link):
r = Request(url=link.url, callback=self._response_downloaded)
r.meta.update(rule=rule, link_text=link.text)
return r
def _requests_to_follow(self, response):
if not isinstance(response, HtmlResponse):
return
@ -59,8 +64,7 @@ class CrawlSpider(Spider):
links = rule.process_links(links)
for link in links:
seen.add(link)
r = Request(url=link.url, callback=self._response_downloaded)
r.meta.update(rule=n, link_text=link.text)
r = self._build_request(n, link)
yield rule.process_request(r)
def _response_downloaded(self, response):