diff --git a/docs/news.rst b/docs/news.rst index 8e29228c5..9cb62edc9 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -17,6 +17,7 @@ Scrapy changes: - dropped support for Twisted 2.5 - added :setting:`REFERER_ENABLED` setting, to control referer middleware - changed default user agent to: ``Scrapy/VERSION (+http://scrapy.org)`` +- removed (undocumented) ``HTMLImageLinkExtractor`` class from ``scrapy.contrib.linkextractors.image`` Scrapyd changes: diff --git a/scrapy/contrib/linkextractors/image.py b/scrapy/contrib/linkextractors/image.py deleted file mode 100644 index a2da66ec0..000000000 --- a/scrapy/contrib/linkextractors/image.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -This module implements the HtmlImageLinkExtractor for extracting -image links only. -""" - -from urlparse import urljoin -from scrapy.link import Link -from scrapy.utils.url import canonicalize_url -from scrapy.utils.python import unicode_to_str, flatten -from scrapy.selector import XPathSelectorList, HtmlXPathSelector - -class HTMLImageLinkExtractor(object): - '''HTMLImageLinkExtractor objects are intended to extract image links from HTML pages - given certain xpath locations. - - These locations can be passed in a list/tuple either when instanciating the LinkExtractor, - or whenever you call extract_links. - If no locations are specified in any of these places, a default pattern '//img' will be used. - If locations are specified when instanciating the LinkExtractor, and also when calling extract_links, - both locations will be used for that call of extract_links''' - - def __init__(self, locations=None, unique=True, canonicalize=True): - self.locations = flatten([locations]) - self.unique = unique - self.canonicalize = canonicalize - - def extract_from_selector(self, selector, encoding, parent=None): - """Extract the links of all the images found in the selector given.""" - links = [] - for img in selector.select('descendant-or-self::img'): - url = self._img_attr(img, "src") - if not url: - continue - text = self._img_attr(img, "alt") or self._img_attr(img, "title") or "" - links.append(Link(unicode_to_str(url, encoding), text=text)) - return links - - def _img_attr(self, img, attr): - """Get the value of the given ``attr`` of ``img`` tag""" - res = img.select("@%s" % attr).extract() - return res[0] if res else None - - def extract_links(self, response): - xs = HtmlXPathSelector(response) - base_url = xs.select('//base/@href').extract() - base_url = urljoin(response.url, base_url[0].encode(response.encoding)) if base_url else response.url - - links = [] - for location in self.locations: - if isinstance(location, basestring): - selectors = xs.select(location) - elif isinstance(location, (XPathSelectorList, HtmlXPathSelector)): - selectors = [location] if isinstance(location, HtmlXPathSelector) else location - else: - continue - - for selector in selectors: - links.extend(self.extract_from_selector(selector, response.encoding)) - - seen, ret = set(), [] - for link in links: - link.url = urljoin(base_url, link.url) - if self.unique: - if link.url in seen: - continue - else: - seen.add(link.url) - if self.canonicalize: - link.url = canonicalize_url(link.url) - ret.append(link) - - return ret - - def matches(self, url): - return False diff --git a/scrapy/tests/sample_data/link_extractor/image_linkextractor.html b/scrapy/tests/sample_data/link_extractor/image_linkextractor.html deleted file mode 100644 index 722240c03..000000000 --- a/scrapy/tests/sample_data/link_extractor/image_linkextractor.html +++ /dev/null @@ -1,21 +0,0 @@ - - - -Sample page with image links for testing HTMLImageLinkExtractor - - -
-
-sample 1 - - -sample 2 - - -sample 3 image -
- - -
- - diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 0697f7102..f986a5bc7 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -4,7 +4,6 @@ import unittest from scrapy.http import HtmlResponse from scrapy.link import Link from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor -from scrapy.contrib.linkextractors.image import HTMLImageLinkExtractor from scrapy.tests import get_testdata class LinkExtractorTestCase(unittest.TestCase): @@ -241,72 +240,6 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) -class HTMLImageLinkExtractorTestCase(unittest.TestCase): - def setUp(self): - body = get_testdata('link_extractor', 'image_linkextractor.html') - self.response = HtmlResponse(url='http://example.com/index', body=body) - - def tearDown(self): - del self.response - - def test_urls_type(self): - '''Test that the resulting urls are regular strings and not a unicode objects''' - lx = HTMLImageLinkExtractor() - links = lx.extract_links(self.response) - self.assertTrue(all(isinstance(link.url, str) for link in links)) - - def test_extraction(self): - '''Test the extractor's behaviour among different situations''' - - lx = HTMLImageLinkExtractor(locations=('//img', )) - links_1 = lx.extract_links(self.response) - self.assertEqual(links_1, - [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), - Link(url='http://example.com/sample2.jpg', text=u'sample 2'), - Link(url='http://example.com/sample4.jpg', text=u'sample 4') ]) - - lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False) - links_2 = lx.extract_links(self.response) - - self.assertEqual(links_2, - [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), - Link(url='http://example.com/sample2.jpg', text=u'sample 2'), - Link(url='http://example.com/sample4.jpg', text=u'sample 4'), - Link(url='http://example.com/sample4.jpg', text=u'sample 4 repetition') ]) - - lx = HTMLImageLinkExtractor(locations=('//div[@id="wrapper"]', )) - links_3 = lx.extract_links(self.response) - self.assertEqual(links_3, - [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), - Link(url='http://example.com/sample2.jpg', text=u'sample 2'), - Link(url='http://example.com/sample4.jpg', text=u'sample 4') ]) - - lx = HTMLImageLinkExtractor(locations=('//a', )) - links_4 = lx.extract_links(self.response) - self.assertEqual(links_4, - [Link(url='http://example.com/sample2.jpg', text=u'sample 2')]) - - def test_extraction_over_selector(self): - from scrapy.selector import HtmlXPathSelector - body = """ - - - - - - Image: - CH29 - - - """ - response = HtmlResponse(url='http://example.com/mypage.html', body=body) - xp = HtmlXPathSelector(response) - lx = HTMLImageLinkExtractor(locations=[xp.select("//img")], canonicalize=False) - - result = lx.extract_links(response) - self.assertTrue(result) - self.assertEquals('http://example.com/images/items/CH29.jpg', result[0].url) - if __name__ == "__main__": unittest.main()