mirror of https://github.com/scrapy/scrapy.git
removed deprecated/undocumented class: HTMLImageLinkExtractor
This commit is contained in:
Pablo Hoffman
parent
13ac6f63eb
commit
97f362d64e
|
|
@ -17,6 +17,7 @@ Scrapy changes:
|
|||
- dropped support for Twisted 2.5
|
||||
- added :setting:`REFERER_ENABLED` setting, to control referer middleware
|
||||
- changed default user agent to: ``Scrapy/VERSION (+http://scrapy.org)``
|
||||
- removed (undocumented) ``HTMLImageLinkExtractor`` class from ``scrapy.contrib.linkextractors.image``
|
||||
|
||||
Scrapyd changes:
|
||||
|
||||
|
|
|
|||
|
|
@ -1,75 +0,0 @@
|
|||
"""
|
||||
This module implements the HtmlImageLinkExtractor for extracting
|
||||
image links only.
|
||||
"""
|
||||
|
||||
from urlparse import urljoin
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.url import canonicalize_url
|
||||
from scrapy.utils.python import unicode_to_str, flatten
|
||||
from scrapy.selector import XPathSelectorList, HtmlXPathSelector
|
||||
|
||||
class HTMLImageLinkExtractor(object):
|
||||
'''HTMLImageLinkExtractor objects are intended to extract image links from HTML pages
|
||||
given certain xpath locations.
|
||||
|
||||
These locations can be passed in a list/tuple either when instanciating the LinkExtractor,
|
||||
or whenever you call extract_links.
|
||||
If no locations are specified in any of these places, a default pattern '//img' will be used.
|
||||
If locations are specified when instanciating the LinkExtractor, and also when calling extract_links,
|
||||
both locations will be used for that call of extract_links'''
|
||||
|
||||
def __init__(self, locations=None, unique=True, canonicalize=True):
|
||||
self.locations = flatten([locations])
|
||||
self.unique = unique
|
||||
self.canonicalize = canonicalize
|
||||
|
||||
def extract_from_selector(self, selector, encoding, parent=None):
|
||||
"""Extract the links of all the images found in the selector given."""
|
||||
links = []
|
||||
for img in selector.select('descendant-or-self::img'):
|
||||
url = self._img_attr(img, "src")
|
||||
if not url:
|
||||
continue
|
||||
text = self._img_attr(img, "alt") or self._img_attr(img, "title") or ""
|
||||
links.append(Link(unicode_to_str(url, encoding), text=text))
|
||||
return links
|
||||
|
||||
def _img_attr(self, img, attr):
|
||||
"""Get the value of the given ``attr`` of ``img`` tag"""
|
||||
res = img.select("@%s" % attr).extract()
|
||||
return res[0] if res else None
|
||||
|
||||
def extract_links(self, response):
|
||||
xs = HtmlXPathSelector(response)
|
||||
base_url = xs.select('//base/@href').extract()
|
||||
base_url = urljoin(response.url, base_url[0].encode(response.encoding)) if base_url else response.url
|
||||
|
||||
links = []
|
||||
for location in self.locations:
|
||||
if isinstance(location, basestring):
|
||||
selectors = xs.select(location)
|
||||
elif isinstance(location, (XPathSelectorList, HtmlXPathSelector)):
|
||||
selectors = [location] if isinstance(location, HtmlXPathSelector) else location
|
||||
else:
|
||||
continue
|
||||
|
||||
for selector in selectors:
|
||||
links.extend(self.extract_from_selector(selector, response.encoding))
|
||||
|
||||
seen, ret = set(), []
|
||||
for link in links:
|
||||
link.url = urljoin(base_url, link.url)
|
||||
if self.unique:
|
||||
if link.url in seen:
|
||||
continue
|
||||
else:
|
||||
seen.add(link.url)
|
||||
if self.canonicalize:
|
||||
link.url = canonicalize_url(link.url)
|
||||
ret.append(link)
|
||||
|
||||
return ret
|
||||
|
||||
def matches(self, url):
|
||||
return False
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
<html>
|
||||
<head>
|
||||
<base href='http://example.com' />
|
||||
<title>Sample page with image links for testing HTMLImageLinkExtractor</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='wrapper'>
|
||||
<div id='subwrapper'>
|
||||
<img src='sample1.jpg' alt='sample 1' />
|
||||
|
||||
<a href='sample2.html' title='sample 2'>
|
||||
<img src='sample2.jpg' alt='sample 2' />
|
||||
</a>
|
||||
|
||||
<a href='sample3.html' title='sample 3'>sample 3 image</a>
|
||||
</div>
|
||||
<img src='sample4.jpg' title='sample 4' />
|
||||
<img src='sample4.jpg' title='sample 4 repetition' />
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -4,7 +4,6 @@ import unittest
|
|||
from scrapy.http import HtmlResponse
|
||||
from scrapy.link import Link
|
||||
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
|
||||
from scrapy.contrib.linkextractors.image import HTMLImageLinkExtractor
|
||||
from scrapy.tests import get_testdata
|
||||
|
||||
class LinkExtractorTestCase(unittest.TestCase):
|
||||
|
|
@ -241,72 +240,6 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
|
|||
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
|
||||
|
||||
|
||||
class HTMLImageLinkExtractorTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
body = get_testdata('link_extractor', 'image_linkextractor.html')
|
||||
self.response = HtmlResponse(url='http://example.com/index', body=body)
|
||||
|
||||
def tearDown(self):
|
||||
del self.response
|
||||
|
||||
def test_urls_type(self):
|
||||
'''Test that the resulting urls are regular strings and not a unicode objects'''
|
||||
lx = HTMLImageLinkExtractor()
|
||||
links = lx.extract_links(self.response)
|
||||
self.assertTrue(all(isinstance(link.url, str) for link in links))
|
||||
|
||||
def test_extraction(self):
|
||||
'''Test the extractor's behaviour among different situations'''
|
||||
|
||||
lx = HTMLImageLinkExtractor(locations=('//img', ))
|
||||
links_1 = lx.extract_links(self.response)
|
||||
self.assertEqual(links_1,
|
||||
[ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
|
||||
Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample4.jpg', text=u'sample 4') ])
|
||||
|
||||
lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False)
|
||||
links_2 = lx.extract_links(self.response)
|
||||
|
||||
self.assertEqual(links_2,
|
||||
[ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
|
||||
Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample4.jpg', text=u'sample 4'),
|
||||
Link(url='http://example.com/sample4.jpg', text=u'sample 4 repetition') ])
|
||||
|
||||
lx = HTMLImageLinkExtractor(locations=('//div[@id="wrapper"]', ))
|
||||
links_3 = lx.extract_links(self.response)
|
||||
self.assertEqual(links_3,
|
||||
[ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
|
||||
Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample4.jpg', text=u'sample 4') ])
|
||||
|
||||
lx = HTMLImageLinkExtractor(locations=('//a', ))
|
||||
links_4 = lx.extract_links(self.response)
|
||||
self.assertEqual(links_4,
|
||||
[Link(url='http://example.com/sample2.jpg', text=u'sample 2')])
|
||||
|
||||
def test_extraction_over_selector(self):
|
||||
from scrapy.selector import HtmlXPathSelector
|
||||
body = """
|
||||
<html>
|
||||
<head>
|
||||
<base href="http://example.com"/>
|
||||
</head>
|
||||
<body>
|
||||
<b>Image: </b>
|
||||
<img src="/images/items/CH29.jpg" width="270" height="270" alt="CH29">
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
response = HtmlResponse(url='http://example.com/mypage.html', body=body)
|
||||
xp = HtmlXPathSelector(response)
|
||||
lx = HTMLImageLinkExtractor(locations=[xp.select("//img")], canonicalize=False)
|
||||
|
||||
result = lx.extract_links(response)
|
||||
self.assertTrue(result)
|
||||
self.assertEquals('http://example.com/images/items/CH29.jpg', result[0].url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue