From 2e1a170563ca8646236da7c48ce526306ac5b1a5 Mon Sep 17 00:00:00 2001 From: Ismael Carnales Date: Tue, 3 Mar 2009 07:42:54 +0000 Subject: [PATCH] added LxmlParserLinkExtractor a new try on link extracting --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40959 --- .../trunk/scrapy/contrib_exp/link/__init__.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/scrapy/trunk/scrapy/contrib_exp/link/__init__.py b/scrapy/trunk/scrapy/contrib_exp/link/__init__.py index e03672241..e3320dcf3 100644 --- a/scrapy/trunk/scrapy/contrib_exp/link/__init__.py +++ b/scrapy/trunk/scrapy/contrib_exp/link/__init__.py @@ -1,5 +1,6 @@ from HTMLParser import HTMLParser from lxml import etree +import lxml.html from scrapy.link import Link from scrapy.utils.python import unique as unique_list @@ -160,3 +161,33 @@ class LinkTarget(object): def close(self): return self.base_url, self.links + + +class LxmlParserLinkExtractor(object): + def __init__(self, tag="a", attr="href", process=None, unique=False): + self.scan_tag = tag if callable(tag) else lambda t: t == tag + self.scan_attr = attr if callable(attr) else lambda a: a == attr + self.process_attr = process if callable(process) else lambda v: v + self.unique = unique + + self.links = [] + + def _extract_links(self, response_text, response_url): + html = lxml.html.fromstring(response_text) + html.make_links_absolute(response_url) + for e, a, l, p in html.iterlinks(): + if self.scan_tag(e.tag): + if self.scan_attr(a): + link = Link(self.process_attr(l), text=e.text) + self.links.append(link) + + links = unique_list(self.links, key=lambda link: link.url) \ + if self.unique else self.links + + return links + + def extract_links(self, response): + # wrapper needed to allow to work directly with text + return self._extract_links(response.body, response.url) + +