diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py index fb5ac2db3..f8a48e797 100644 --- a/scrapy/contrib/linkextractors/lxmlhtml.py +++ b/scrapy/contrib/linkextractors/lxmlhtml.py @@ -34,8 +34,6 @@ class LxmlParserLinkExtractor(object): self.process_attr = process if callable(process) else lambda v: v self.unique = unique - self.links = [] - def _iter_links(self, document): for el in document.iter(etree.Element): tag = _nons(el.tag) @@ -46,6 +44,7 @@ class LxmlParserLinkExtractor(object): yield (el, attrib, attribs[attrib]) def _extract_links(self, selector, response_url, response_encoding, base_url): + links = [] # hacky way to get the underlying lxml parsed document for el, attr, attr_val in self._iter_links(selector._root): if self.scan_tag(el.tag) and self.scan_attr(attr): @@ -60,12 +59,10 @@ class LxmlParserLinkExtractor(object): url = urljoin(response_url, url) link = Link(url, _collect_string_content(el) or u'', nofollow=True if el.get('rel') == 'nofollow' else False) - self.links.append(link) + links.append(link) - links = unique_list(self.links, key=lambda link: link.url) \ - if self.unique else self.links - - return links + return unique_list(links, key=lambda link: link.url) \ + if self.unique else links def extract_links(self, response): html = Selector(response)