diff --git a/scrapy/contrib/linkextractors/image.py b/scrapy/contrib/linkextractors/image.py index 741060769..6e129f392 100644 --- a/scrapy/contrib/linkextractors/image.py +++ b/scrapy/contrib/linkextractors/image.py @@ -7,7 +7,7 @@ from urlparse import urljoin from scrapy.link import Link from scrapy.utils.url import canonicalize_url from scrapy.utils.python import unicode_to_str, flatten -from scrapy.selector.libxml2sel import XPathSelectorList, HtmlXPathSelector +from scrapy.selector import XPathSelectorList, HtmlXPathSelector class HTMLImageLinkExtractor(object): '''HTMLImageLinkExtractor objects are intended to extract image links from HTML pages @@ -25,28 +25,26 @@ class HTMLImageLinkExtractor(object): self.canonicalize = canonicalize def extract_from_selector(self, selector, encoding, parent=None): - ret = [] - def _add_link(url_sel, alt_sel=None): - url = flatten([url_sel.extract()]) - alt = flatten([alt_sel.extract()]) if alt_sel else (u'', ) - if url: - ret.append(Link(unicode_to_str(url[0], encoding), alt[0])) + """Extract the links of all the images found in the selector given.""" - if selector.xmlNode.type == 'element': - if selector.xmlNode.name == 'img': - _add_link(selector.select('@src'), selector.select('@alt') or \ - selector.select('@title')) - else: - children = selector.select('child::*') - if len(children): - for child in children: - ret.extend(self.extract_from_selector(child, encoding, parent=selector)) - elif selector.xmlNode.name == 'a' and not parent: - _add_link(selector.select('@href'), selector.select('@title')) - else: - _add_link(selector) + selectors = [selector] if selector.select("local-name()").re("^img$") \ + else selector.select(".//img") + + def _img_attr(img, attr): + """Helper to get the value of the given ``attr`` of the ``img`` + selector""" + res = img.select("@%s" % attr).extract() + return res[0] if res else None + + links = [] + for img in selectors: + url = _img_attr(img, "src") + text = _img_attr(img, "alt") or _img_attr(img, "title") or "" + if not url: + continue + links.append(Link(unicode_to_str(url, encoding), text=text)) + return links - return ret def extract_links(self, response): xs = HtmlXPathSelector(response) diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 6133d4c6d..0697f7102 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -267,6 +267,7 @@ class HTMLImageLinkExtractorTestCase(unittest.TestCase): lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False) links_2 = lx.extract_links(self.response) + self.assertEqual(links_2, [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), Link(url='http://example.com/sample2.jpg', text=u'sample 2'), @@ -283,8 +284,29 @@ class HTMLImageLinkExtractorTestCase(unittest.TestCase): lx = HTMLImageLinkExtractor(locations=('//a', )) links_4 = lx.extract_links(self.response) self.assertEqual(links_4, - [ Link(url='http://example.com/sample2.jpg', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3') ]) + [Link(url='http://example.com/sample2.jpg', text=u'sample 2')]) + + def test_extraction_over_selector(self): + from scrapy.selector import HtmlXPathSelector + body = """ + + + + + + Image: + CH29 + + + """ + response = HtmlResponse(url='http://example.com/mypage.html', body=body) + xp = HtmlXPathSelector(response) + lx = HTMLImageLinkExtractor(locations=[xp.select("//img")], canonicalize=False) + + result = lx.extract_links(response) + self.assertTrue(result) + self.assertEquals('http://example.com/images/items/CH29.jpg', result[0].url) + if __name__ == "__main__": unittest.main()