From 35c1dcdbc2eeaace7fce5e58e22f395625b87422 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Wed, 5 Aug 2015 19:47:16 -0300 Subject: [PATCH] use response.selector in link extractors instead of instantiating new Selector --- scrapy/linkextractors/lxmlhtml.py | 8 +++----- scrapy/linkextractors/sgml.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index c952a5f83..7c41a88ff 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -65,9 +65,8 @@ class LxmlParserLinkExtractor(object): if self.unique else links def extract_links(self, response): - html = Selector(response) base_url = get_base_url(response) - return self._extract_links(html, response.url, response.encoding, base_url) + return self._extract_links(response.selector, response.url, response.encoding, base_url) def _process_links(self, links): """ Normalize and filter extracted links @@ -95,14 +94,13 @@ class LxmlLinkExtractor(FilteringLinkExtractor): canonicalize=canonicalize, deny_extensions=deny_extensions) def extract_links(self, response): - html = Selector(response) base_url = get_base_url(response) if self.restrict_xpaths: docs = [subdoc for x in self.restrict_xpaths - for subdoc in html.xpath(x)] + for subdoc in response.xpath(x)] else: - docs = [html] + docs = [response.selector] all_links = [] for doc in docs: links = self._extract_links(doc, response.url, response.encoding, base_url) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index e4c2c274f..d045baa24 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -127,7 +127,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor): def extract_links(self, response): base_url = None if self.restrict_xpaths: - sel = Selector(response) + sel = response.selector base_url = get_base_url(response) body = u''.join(f for x in self.restrict_xpaths