diff --git a/scrapy/trunk/scrapy/contrib_exp/link/__init__.py b/scrapy/trunk/scrapy/contrib_exp/link/__init__.py index 41efb4420..e03672241 100644 --- a/scrapy/trunk/scrapy/contrib_exp/link/__init__.py +++ b/scrapy/trunk/scrapy/contrib_exp/link/__init__.py @@ -1,10 +1,11 @@ from HTMLParser import HTMLParser +from lxml import etree from scrapy.link import Link from scrapy.utils.python import unique as unique_list from scrapy.utils.url import safe_url_string, urljoin_rfc as urljoin -class LinkExtractor(HTMLParser): +class HtmlParserLinkExtractor(HTMLParser): """LinkExtractor are used to extract links from web pages. They are instantiated and later "applied" to a Response using the extract_links @@ -26,10 +27,10 @@ class LinkExtractor(HTMLParser): * a tag name which is used to search for links (defaults to "a") * a function which receives a tag name and returns whether to scan it * attr (string or function) - * an attribute name which is used to search for links (defaults to "href") - * a function which receives an attribute name and returns whether to scan it + * an attrsute name which is used to search for links (defaults to "href") + * a function which receives an attrsute name and returns whether to scan it * process (funtion) - * a function wich receives the attribute value before assigning it + * a function wich receives the attrsute value before assigning it * unique - if True the same urls won't be extracted twice, otherwise the same urls will be extracted multiple times (with potentially different link texts) """ @@ -94,3 +95,68 @@ class LinkExtractor(HTMLParser): it doesn't contain any patterns""" return True + +class LxmlLinkExtractor(object): + def __init__(self, tag="a", attr="href", process=None, unique=False): + scan_tag = tag if callable(tag) else lambda t: t == tag + scan_attr = attr if callable(attr) else lambda a: a == attr + process_attr = process if callable(process) else lambda v: v + + self.unique = unique + + target = LinkTarget(scan_tag, scan_attr, process_attr) + self.parser = etree.HTMLParser(target=target) + + + def _extract_links(self, response_text, response_url, response_encoding): + self.base_url, self.links = etree.HTML(response_text, self.parser) + + links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links + + ret = [] + base_url = self.base_url if self.base_url else response_url + for link in links: + link.url = urljoin(base_url, link.url) + link.url = safe_url_string(link.url, response_encoding) + link.text = link.text.decode(response_encoding) + ret.append(link) + + return ret + + def extract_links(self, response): + # wrapper needed to allow to work directly with text + return self._extract_links(response.body, response.url, + response.encoding) + + +class LinkTarget(object): + def __init__(self, scan_tag, scan_attr, process_attr): + self.scan_tag = scan_tag + self.scan_attr = scan_attr + self.process_attr = process_attr + + self.base_url = None + self.links = [] + + self.current_link = None + + def start(self, tag, attrs): + if tag == 'base': + self.base_url = dict(attrs).get('href') + if self.scan_tag(tag): + for attr, value in attrs.iteritems(): + if self.scan_attr(attr): + url = self.process_attr(value) + link = Link(url=url) + self.links.append(link) + self.current_link = link + + def end(self, tag): + self.current_link = None + + def data(self, data): + if self.current_link and not self.current_link.text: + self.current_link.text = data.strip() + + def close(self): + return self.base_url, self.links