diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py
index f8a48e797..efe24b835 100644
--- a/scrapy/contrib/linkextractors/lxmlhtml.py
+++ b/scrapy/contrib/linkextractors/lxmlhtml.py
@@ -36,30 +36,30 @@ class LxmlParserLinkExtractor(object):
def _iter_links(self, document):
for el in document.iter(etree.Element):
- tag = _nons(el.tag)
- if not self.scan_tag(el.tag):
+ if not self.scan_tag(_nons(el.tag)):
continue
attribs = el.attrib
for attrib in attribs:
+ if not self.scan_attr(attrib):
+ continue
yield (el, attrib, attribs[attrib])
def _extract_links(self, selector, response_url, response_encoding, base_url):
links = []
# hacky way to get the underlying lxml parsed document
for el, attr, attr_val in self._iter_links(selector._root):
- if self.scan_tag(el.tag) and self.scan_attr(attr):
- # pseudo _root.make_links_absolute(base_url)
- attr_val = urljoin(base_url, attr_val)
- url = self.process_attr(attr_val)
- if url is None:
- continue
- if isinstance(url, unicode):
- url = url.encode(response_encoding)
- # to fix relative links after process_value
- url = urljoin(response_url, url)
- link = Link(url, _collect_string_content(el) or u'',
- nofollow=True if el.get('rel') == 'nofollow' else False)
- links.append(link)
+ # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
+ attr_val = urljoin(base_url, attr_val)
+ url = self.process_attr(attr_val)
+ if url is None:
+ continue
+ if isinstance(url, unicode):
+ url = url.encode(response_encoding)
+ # to fix relative links after process_value
+ url = urljoin(response_url, url)
+ link = Link(url, _collect_string_content(el) or u'',
+ nofollow=True if el.get('rel') == 'nofollow' else False)
+ links.append(link)
return unique_list(links, key=lambda link: link.url) \
if self.unique else links
diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py
index 27d73e271..3609e4b80 100644
--- a/scrapy/tests/test_contrib_linkextractors.py
+++ b/scrapy/tests/test_contrib_linkextractors.py
@@ -1,7 +1,7 @@
import re
import unittest
from scrapy.contrib.linkextractors.regex import RegexLinkExtractor
-from scrapy.http import HtmlResponse
+from scrapy.http import HtmlResponse, XmlResponse
from scrapy.link import Link
from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
@@ -419,6 +419,51 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
])
+ def test_xhtml(self):
+ xhtml = """
+
+
+
+
+ XHTML document title
+
+
+
+
+
+
+
+
+ """
+
+ response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
+
+ lx = self.extractor_cls()
+ self.assertEqual(lx.extract_links(response),
+ [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+ Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+ Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+ Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
+ )
+
+ response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
+
+ lx = self.extractor_cls()
+ self.assertEqual(lx.extract_links(response),
+ [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+ Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+ Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+ Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
+ )
class LxmlLinkExtractorTestCase(SgmlLinkExtractorTestCase):