RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use response.selector in link extractors instead of instantiating new Selector

This commit is contained in:
Elias Dorneles 2015-08-05 19:47:16 -03:00
parent 17d7347a36
commit 35c1dcdbc2
2 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/linkextractors/lxmlhtml.py
View File

@ -65,9 +65,8 @@ class LxmlParserLinkExtractor(object):
if self.unique else links
def extract_links(self, response):
html = Selector(response)
base_url = get_base_url(response)
return self._extract_links(html, response.url, response.encoding, base_url)
return self._extract_links(response.selector, response.url, response.encoding, base_url)
def _process_links(self, links):
""" Normalize and filter extracted links
@ -95,14 +94,13 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
canonicalize=canonicalize, deny_extensions=deny_extensions)
def extract_links(self, response):
html = Selector(response)
base_url = get_base_url(response)
if self.restrict_xpaths:
docs = [subdoc
for x in self.restrict_xpaths
for subdoc in html.xpath(x)]
for subdoc in response.xpath(x)]
else:
docs = [html]
docs = [response.selector]
all_links = []
for doc in docs:
links = self._extract_links(doc, response.url, response.encoding, base_url)

2
scrapy/linkextractors/sgml.py
View File

@ -127,7 +127,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
def extract_links(self, response):
base_url = None
if self.restrict_xpaths:
sel = Selector(response)
sel = response.selector
base_url = get_base_url(response)
body = u''.join(f
for x in self.restrict_xpaths