mirror of https://github.com/scrapy/scrapy.git
cleaned up lxml-based link extractors, and left one of them
--HG-- rename : scrapy/contrib/linkextractors/lxmlparser.py => scrapy/contrib/linkextractors/lxmlhtml.py
This commit is contained in:
Pablo Hoffman
parent
c6f29f02c4
commit
fbf0e9ef43
|
|
@ -0,0 +1,36 @@
|
|||
"""
|
||||
Link extractor based on lxml.html
|
||||
"""
|
||||
|
||||
import lxml.html
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.python import unique as unique_list
|
||||
|
||||
class LxmlParserLinkExtractor(object):
|
||||
def __init__(self, tag="a", attr="href", process=None, unique=False):
|
||||
self.scan_tag = tag if callable(tag) else lambda t: t == tag
|
||||
self.scan_attr = attr if callable(attr) else lambda a: a == attr
|
||||
self.process_attr = process if callable(process) else lambda v: v
|
||||
self.unique = unique
|
||||
|
||||
self.links = []
|
||||
|
||||
def _extract_links(self, response_text, response_url):
|
||||
html = lxml.html.fromstring(response_text)
|
||||
html.make_links_absolute(response_url)
|
||||
for e, a, l, p in html.iterlinks():
|
||||
if self.scan_tag(e.tag):
|
||||
if self.scan_attr(a):
|
||||
link = Link(self.process_attr(l), text=e.text)
|
||||
self.links.append(link)
|
||||
|
||||
links = unique_list(self.links, key=lambda link: link.url) \
|
||||
if self.unique else self.links
|
||||
|
||||
return links
|
||||
|
||||
def extract_links(self, response):
|
||||
return self._extract_links(response.body, response.url)
|
||||
|
||||
|
||||
|
|
@ -1,111 +0,0 @@
|
|||
"""
|
||||
lxml-based link extractors (experimental)
|
||||
|
||||
NOTE: The ideal name for this module would be `lxml`, but that's not possible
|
||||
because it collides with the lxml library module.
|
||||
"""
|
||||
|
||||
from lxml import etree
|
||||
import lxml.html
|
||||
from w3lib.url import safe_url_string, urljoin_rfc
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.python import unique as unique_list, str_to_unicode
|
||||
|
||||
class LxmlLinkExtractor(object):
|
||||
def __init__(self, tag="a", attr="href", process=None, unique=False):
|
||||
scan_tag = tag if callable(tag) else lambda t: t == tag
|
||||
scan_attr = attr if callable(attr) else lambda a: a == attr
|
||||
process_attr = process if callable(process) else lambda v: v
|
||||
|
||||
self.unique = unique
|
||||
|
||||
target = LinkTarget(scan_tag, scan_attr, process_attr)
|
||||
self.parser = etree.HTMLParser(target=target)
|
||||
|
||||
def _extract_links(self, response_text, response_url, response_encoding):
|
||||
self.base_url, self.links = etree.HTML(response_text, self.parser)
|
||||
|
||||
links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
|
||||
|
||||
ret = []
|
||||
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
|
||||
for link in links:
|
||||
link.url = urljoin_rfc(base_url, link.url, response_encoding)
|
||||
link.url = safe_url_string(link.url, response_encoding)
|
||||
link.text = str_to_unicode(link.text, response_encoding, errors='replace')
|
||||
ret.append(link)
|
||||
|
||||
return ret
|
||||
|
||||
def extract_links(self, response):
|
||||
# wrapper needed to allow to work directly with text
|
||||
return self._extract_links(response.body, response.url,
|
||||
response.encoding)
|
||||
|
||||
def matches(self, url):
|
||||
"""This extractor matches with any url, since it doesn't contain any patterns"""
|
||||
return True
|
||||
|
||||
|
||||
class LinkTarget(object):
|
||||
def __init__(self, scan_tag, scan_attr, process_attr):
|
||||
self.scan_tag = scan_tag
|
||||
self.scan_attr = scan_attr
|
||||
self.process_attr = process_attr
|
||||
|
||||
self.base_url = None
|
||||
self.links = []
|
||||
|
||||
self.current_link = None
|
||||
|
||||
def start(self, tag, attrs):
|
||||
if tag == 'base':
|
||||
self.base_url = dict(attrs).get('href')
|
||||
if self.scan_tag(tag):
|
||||
for attr, value in attrs.iteritems():
|
||||
if self.scan_attr(attr):
|
||||
url = self.process_attr(value)
|
||||
link = Link(url=url)
|
||||
self.links.append(link)
|
||||
self.current_link = link
|
||||
|
||||
def end(self, tag):
|
||||
self.current_link = None
|
||||
|
||||
def data(self, data):
|
||||
if self.current_link and not self.current_link.text:
|
||||
self.current_link.text = data.strip()
|
||||
|
||||
def close(self):
|
||||
return self.base_url, self.links
|
||||
|
||||
|
||||
class LxmlParserLinkExtractor(object):
|
||||
def __init__(self, tag="a", attr="href", process=None, unique=False):
|
||||
self.scan_tag = tag if callable(tag) else lambda t: t == tag
|
||||
self.scan_attr = attr if callable(attr) else lambda a: a == attr
|
||||
self.process_attr = process if callable(process) else lambda v: v
|
||||
self.unique = unique
|
||||
|
||||
self.links = []
|
||||
|
||||
def _extract_links(self, response_text, response_url):
|
||||
html = lxml.html.fromstring(response_text)
|
||||
html.make_links_absolute(response_url)
|
||||
for e, a, l, p in html.iterlinks():
|
||||
if self.scan_tag(e.tag):
|
||||
if self.scan_attr(a):
|
||||
link = Link(self.process_attr(l), text=e.text)
|
||||
self.links.append(link)
|
||||
|
||||
links = unique_list(self.links, key=lambda link: link.url) \
|
||||
if self.unique else self.links
|
||||
|
||||
return links
|
||||
|
||||
def extract_links(self, response):
|
||||
# wrapper needed to allow to work directly with text
|
||||
return self._extract_links(response.body, response.url)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue