From 54d2b110f103770edd0c3199c4319f2e1223cbd4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 10 Jul 2014 13:04:21 +0200 Subject: [PATCH] Extract links from XHTML documents with MIME-Type "application/xml" "application/xhtml+xml" is already interpreted as HTML and link extractor is fine with it to extract links. Only for XML documents can the namespaces in tags be an issue. Fixes #780 Do both tag and attribute tests in _iter_links() method --- scrapy/contrib/linkextractors/lxmlhtml.py | 30 ++++++------- scrapy/tests/test_contrib_linkextractors.py | 47 ++++++++++++++++++++- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py index f8a48e797..efe24b835 100644 --- a/scrapy/contrib/linkextractors/lxmlhtml.py +++ b/scrapy/contrib/linkextractors/lxmlhtml.py @@ -36,30 +36,30 @@ class LxmlParserLinkExtractor(object): def _iter_links(self, document): for el in document.iter(etree.Element): - tag = _nons(el.tag) - if not self.scan_tag(el.tag): + if not self.scan_tag(_nons(el.tag)): continue attribs = el.attrib for attrib in attribs: + if not self.scan_attr(attrib): + continue yield (el, attrib, attribs[attrib]) def _extract_links(self, selector, response_url, response_encoding, base_url): links = [] # hacky way to get the underlying lxml parsed document for el, attr, attr_val in self._iter_links(selector._root): - if self.scan_tag(el.tag) and self.scan_attr(attr): - # pseudo _root.make_links_absolute(base_url) - attr_val = urljoin(base_url, attr_val) - url = self.process_attr(attr_val) - if url is None: - continue - if isinstance(url, unicode): - url = url.encode(response_encoding) - # to fix relative links after process_value - url = urljoin(response_url, url) - link = Link(url, _collect_string_content(el) or u'', - nofollow=True if el.get('rel') == 'nofollow' else False) - links.append(link) + # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) + attr_val = urljoin(base_url, attr_val) + url = self.process_attr(attr_val) + if url is None: + continue + if isinstance(url, unicode): + url = url.encode(response_encoding) + # to fix relative links after process_value + url = urljoin(response_url, url) + link = Link(url, _collect_string_content(el) or u'', + nofollow=True if el.get('rel') == 'nofollow' else False) + links.append(link) return unique_list(links, key=lambda link: link.url) \ if self.unique else links diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 27d73e271..3609e4b80 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -1,7 +1,7 @@ import re import unittest from scrapy.contrib.linkextractors.regex import RegexLinkExtractor -from scrapy.http import HtmlResponse +from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor @@ -419,6 +419,51 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) ]) + def test_xhtml(self): + xhtml = """ + + + + + XHTML document title + + +
+

About us

+
+
+

Follow this link

+
+
+

Dont follow this one

+
+
+

Choose to follow or not

+
+ + + """ + + response = HtmlResponse("http://example.com/index.xhtml", body=xhtml) + + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)] + ) + + response = XmlResponse("http://example.com/index.xhtml", body=xhtml) + + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)] + ) class LxmlLinkExtractorTestCase(SgmlLinkExtractorTestCase):