RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2562 from terut/feature/build_request 

[MRG+1] Separate building request from _requests_to_follow in CrawlSpider
This commit is contained in:
Daniel Graña 2017-02-20 11:12:10 -03:00 committed by GitHub
parent 322fd68e4c 5b31dfe3c9
commit b2d66dc1f0
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/spiders/crawl.py
View File

@ -48,6 +48,11 @@ class CrawlSpider(Spider):
def process_results(self, response, results):
return results
def _build_request(self, rule, link):
r = Request(url=link.url, callback=self._response_downloaded)
r.meta.update(rule=rule, link_text=link.text)
return r
def _requests_to_follow(self, response):
if not isinstance(response, HtmlResponse):
return
@ -59,8 +64,7 @@ class CrawlSpider(Spider):
links = rule.process_links(links)
for link in links:
seen.add(link)
r = Request(url=link.url, callback=self._response_downloaded)
r.meta.update(rule=n, link_text=link.text)
r = self._build_request(n, link)
yield rule.process_request(r)
def _response_downloaded(self, response):