From fbf0e9ef438306cb44ab32d01295b486956cc679 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 2 Aug 2011 15:10:25 -0300 Subject: [PATCH] cleaned up lxml-based link extractors, and left one of them --HG-- rename : scrapy/contrib/linkextractors/lxmlparser.py => scrapy/contrib/linkextractors/lxmlhtml.py --- scrapy/contrib/linkextractors/lxmlhtml.py | 36 +++++++ scrapy/contrib/linkextractors/lxmlparser.py | 111 -------------------- 2 files changed, 36 insertions(+), 111 deletions(-) create mode 100644 scrapy/contrib/linkextractors/lxmlhtml.py delete mode 100644 scrapy/contrib/linkextractors/lxmlparser.py diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py new file mode 100644 index 000000000..826c76227 --- /dev/null +++ b/scrapy/contrib/linkextractors/lxmlhtml.py @@ -0,0 +1,36 @@ +""" +Link extractor based on lxml.html +""" + +import lxml.html + +from scrapy.link import Link +from scrapy.utils.python import unique as unique_list + +class LxmlParserLinkExtractor(object): + def __init__(self, tag="a", attr="href", process=None, unique=False): + self.scan_tag = tag if callable(tag) else lambda t: t == tag + self.scan_attr = attr if callable(attr) else lambda a: a == attr + self.process_attr = process if callable(process) else lambda v: v + self.unique = unique + + self.links = [] + + def _extract_links(self, response_text, response_url): + html = lxml.html.fromstring(response_text) + html.make_links_absolute(response_url) + for e, a, l, p in html.iterlinks(): + if self.scan_tag(e.tag): + if self.scan_attr(a): + link = Link(self.process_attr(l), text=e.text) + self.links.append(link) + + links = unique_list(self.links, key=lambda link: link.url) \ + if self.unique else self.links + + return links + + def extract_links(self, response): + return self._extract_links(response.body, response.url) + + diff --git a/scrapy/contrib/linkextractors/lxmlparser.py b/scrapy/contrib/linkextractors/lxmlparser.py deleted file mode 100644 index 21b06cae8..000000000 --- a/scrapy/contrib/linkextractors/lxmlparser.py +++ /dev/null @@ -1,111 +0,0 @@ -""" -lxml-based link extractors (experimental) - -NOTE: The ideal name for this module would be `lxml`, but that's not possible -because it collides with the lxml library module. -""" - -from lxml import etree -import lxml.html -from w3lib.url import safe_url_string, urljoin_rfc - -from scrapy.link import Link -from scrapy.utils.python import unique as unique_list, str_to_unicode - -class LxmlLinkExtractor(object): - def __init__(self, tag="a", attr="href", process=None, unique=False): - scan_tag = tag if callable(tag) else lambda t: t == tag - scan_attr = attr if callable(attr) else lambda a: a == attr - process_attr = process if callable(process) else lambda v: v - - self.unique = unique - - target = LinkTarget(scan_tag, scan_attr, process_attr) - self.parser = etree.HTMLParser(target=target) - - def _extract_links(self, response_text, response_url, response_encoding): - self.base_url, self.links = etree.HTML(response_text, self.parser) - - links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links - - ret = [] - base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url - for link in links: - link.url = urljoin_rfc(base_url, link.url, response_encoding) - link.url = safe_url_string(link.url, response_encoding) - link.text = str_to_unicode(link.text, response_encoding, errors='replace') - ret.append(link) - - return ret - - def extract_links(self, response): - # wrapper needed to allow to work directly with text - return self._extract_links(response.body, response.url, - response.encoding) - - def matches(self, url): - """This extractor matches with any url, since it doesn't contain any patterns""" - return True - - -class LinkTarget(object): - def __init__(self, scan_tag, scan_attr, process_attr): - self.scan_tag = scan_tag - self.scan_attr = scan_attr - self.process_attr = process_attr - - self.base_url = None - self.links = [] - - self.current_link = None - - def start(self, tag, attrs): - if tag == 'base': - self.base_url = dict(attrs).get('href') - if self.scan_tag(tag): - for attr, value in attrs.iteritems(): - if self.scan_attr(attr): - url = self.process_attr(value) - link = Link(url=url) - self.links.append(link) - self.current_link = link - - def end(self, tag): - self.current_link = None - - def data(self, data): - if self.current_link and not self.current_link.text: - self.current_link.text = data.strip() - - def close(self): - return self.base_url, self.links - - -class LxmlParserLinkExtractor(object): - def __init__(self, tag="a", attr="href", process=None, unique=False): - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.process_attr = process if callable(process) else lambda v: v - self.unique = unique - - self.links = [] - - def _extract_links(self, response_text, response_url): - html = lxml.html.fromstring(response_text) - html.make_links_absolute(response_url) - for e, a, l, p in html.iterlinks(): - if self.scan_tag(e.tag): - if self.scan_attr(a): - link = Link(self.process_attr(l), text=e.text) - self.links.append(link) - - links = unique_list(self.links, key=lambda link: link.url) \ - if self.unique else self.links - - return links - - def extract_links(self, response): - # wrapper needed to allow to work directly with text - return self._extract_links(response.body, response.url) - -