RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix HTMLImageLinkExtractor to work with libxml2 and lxml selectors

This commit is contained in:
Andrés Moreira 2012-04-18 18:05:44 -03:00
parent 30ddbf624e
commit e24107feb8
2 changed files with 43 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
scrapy/contrib/linkextractors/image.py
View File

@ -7,7 +7,7 @@ from urlparse import urljoin
from scrapy.link import Link
from scrapy.utils.url import canonicalize_url
from scrapy.utils.python import unicode_to_str, flatten
from scrapy.selector.libxml2sel import XPathSelectorList, HtmlXPathSelector
from scrapy.selector import XPathSelectorList, HtmlXPathSelector
class HTMLImageLinkExtractor(object):
'''HTMLImageLinkExtractor objects are intended to extract image links from HTML pages
@ -25,28 +25,26 @@ class HTMLImageLinkExtractor(object):
self.canonicalize = canonicalize
def extract_from_selector(self, selector, encoding, parent=None):
ret = []
def _add_link(url_sel, alt_sel=None):
url = flatten([url_sel.extract()])
alt = flatten([alt_sel.extract()]) if alt_sel else (u'', )
if url:
ret.append(Link(unicode_to_str(url[0], encoding), alt[0]))
"""Extract the links of all the images found in the selector given."""
if selector.xmlNode.type == 'element':
if selector.xmlNode.name == 'img':
_add_link(selector.select('@src'), selector.select('@alt') or \
selector.select('@title'))
else:
children = selector.select('child::*')
if len(children):
for child in children:
ret.extend(self.extract_from_selector(child, encoding, parent=selector))
elif selector.xmlNode.name == 'a' and not parent:
_add_link(selector.select('@href'), selector.select('@title'))
else:
_add_link(selector)
selectors = [selector] if selector.select("local-name()").re("^img$") \
else selector.select(".//img")
def _img_attr(img, attr):
"""Helper to get the value of the given ``attr`` of the ``img``
selector"""
res = img.select("@%s" % attr).extract()
return res[0] if res else None
links = []
for img in selectors:
url = _img_attr(img, "src")
text = _img_attr(img, "alt") or _img_attr(img, "title") or ""
if not url:
continue
links.append(Link(unicode_to_str(url, encoding), text=text))
return links
return ret
def extract_links(self, response):
xs = HtmlXPathSelector(response)

26
scrapy/tests/test_contrib_linkextractors.py
View File

@ -267,6 +267,7 @@ class HTMLImageLinkExtractorTestCase(unittest.TestCase):
lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False)
links_2 = lx.extract_links(self.response)
self.assertEqual(links_2,
[ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
@ -283,8 +284,29 @@ class HTMLImageLinkExtractorTestCase(unittest.TestCase):
lx = HTMLImageLinkExtractor(locations=('//a', ))
links_4 = lx.extract_links(self.response)
self.assertEqual(links_4,
[ Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3') ])
[Link(url='http://example.com/sample2.jpg', text=u'sample 2')])
def test_extraction_over_selector(self):
from scrapy.selector import HtmlXPathSelector
body = """
<html>
<head>
<base href="http://example.com"/>
</head>
<body>
<b>Image: </b>
<img src="/images/items/CH29.jpg" width="270" height="270" alt="CH29">
</body>
</html>
"""
response = HtmlResponse(url='http://example.com/mypage.html', body=body)
xp = HtmlXPathSelector(response)
lx = HTMLImageLinkExtractor(locations=[xp.select("//img")], canonicalize=False)
result = lx.extract_links(response)
self.assertTrue(result)
self.assertEquals('http://example.com/images/items/CH29.jpg', result[0].url)
if __name__ == "__main__":
unittest.main()