diff --git a/scrapy/trunk/scrapy/link/__init__.py b/scrapy/trunk/scrapy/link/__init__.py index bdcf209d9..58ca1f641 100644 --- a/scrapy/trunk/scrapy/link/__init__.py +++ b/scrapy/trunk/scrapy/link/__init__.py @@ -91,3 +91,6 @@ class Link(object): def __eq__(self, other): return self.url == other.url and self.text == other.text + + def __repr__(self): + return '' % (self.url, self.text) diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/trunk/scrapy/link/extractors.py index 4a26d526c..48d52d32a 100644 --- a/scrapy/trunk/scrapy/link/extractors.py +++ b/scrapy/trunk/scrapy/link/extractors.py @@ -5,11 +5,13 @@ This module provides some LinkExtractors, which extend that base LinkExtractor """ import re +import urlparse -from scrapy.link import LinkExtractor +from scrapy.link import LinkExtractor, Link from scrapy.utils.url import canonicalize_url, url_is_from_any_domain from scrapy.utils.response import new_response_from_xpaths -from scrapy.utils.misc import dict_updatedefault +from scrapy.utils.python import unicode_to_str +from scrapy.xpath.selector import HtmlXPathSelector _re_type = type(re.compile("", 0)) @@ -83,3 +85,63 @@ class RegexLinkExtractor(LinkExtractor): allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True] denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else [] return any(allowed) and not any(denied) + +class ImageLinkExtractor(object): + def __init__(self, locations=None): + self.locations = tuple(locations) if hasattr(locations, '__iter__') else tuple() + + def extract_from_selector(self, selector, parent=None): + ret = [] + def _add_link(url_sel, alt_sel=None): + url = url_sel.extract() + alt = alt_sel.extract() if alt_sel else ('', ) + if url: + ret.append(Link(unicode_to_str(url[0]), alt[0])) + + if selector.xmlNode.type == 'element': + if selector.xmlNode.name == 'img': + _add_link(selector.x('@src'), selector.x('@alt') or selector.x('@title')) + else: + children = selector.x('child::*') + if len(children): + for child in children: + ret.extend(self.extract_from_selector(child, parent=selector)) + elif selector.xmlNode.name == 'a' and not parent: + _add_link(selector.x('@href'), selector.x('@title')) + else: + _add_link(selector) + + return ret + + def extract_links(self, response, locations=None, unique=True): + xs = HtmlXPathSelector(response) + base_url = xs.x('//base/@href').extract() + base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url) + + if hasattr(locations, '__iter__'): + locations = self.locations + tuple(locations) + elif self.locations: + locations = self.locations + else: + locations = ('//img', ) + + links = [] + for location in locations: + selector_res = xs.x(location) + for selector in selector_res: + links.extend(self.extract_from_selector(selector)) + + seen, ret = set(), [] + for link in links: + link.url = urlparse.urljoin(base_url, link.url) + if unique: + if link.url in seen: + continue + else: + seen.add(link.url) + ret.append(link) + + return ret + + def matches(self, url): + return False diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/trunk/scrapy/tests/test_link.py index 4fe992753..43f95d3d0 100644 --- a/scrapy/trunk/scrapy/tests/test_link.py +++ b/scrapy/trunk/scrapy/tests/test_link.py @@ -1,8 +1,9 @@ +import os import unittest -from scrapy.http import Response +from scrapy.http.response import Response, ResponseBody from scrapy.link import LinkExtractor, Link -from scrapy.link.extractors import RegexLinkExtractor +from scrapy.link.extractors import RegexLinkExtractor, ImageLinkExtractor class LinkExtractorTestCase(unittest.TestCase): def test_basic(self): @@ -63,5 +64,44 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.matches('http://blah2.com/blah1'), False) self.assertEqual(lx.matches('http://blah2.com/blah2'), False) +class ImageLinkExtractorTestCase(unittest.TestCase): + def setUp(self): + body = open(os.path.join(os.path.dirname(__file__), 'sample_data/image_linkextractor.html'), 'r').read() + self.response = Response(url='http://examplesite.com/index', domain='examplesite.com', body=ResponseBody(body)) + + def test_urls_type(self): + '''Test that the resulting urls are regular strings and not a unicode objects''' + lx = ImageLinkExtractor() + links = lx.extract_links(self.response) + self.assertTrue(all(isinstance(link.url, str) for link in links)) + + def test_extraction(self): + '''Test the extractor's behaviour among different situations''' + lx = ImageLinkExtractor() + + links_1 = lx.extract_links(self.response) # using default locations (//img) + self.assertEqual(links_1, + [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'), + Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), + Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ]) + + links_2 = lx.extract_links(self.response, unique=False) # using default locations and unique=False + self.assertEqual(links_2, + [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'), + Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), + Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4'), + Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4 repetition') ]) + + links_3 = lx.extract_links(self.response, locations=('//div[@id="wrapper"]', )) + self.assertEqual(links_3, + [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'), + Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), + Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ]) + + links_4 = lx.extract_links(self.response, locations=('//a', )) + self.assertEqual(links_4, + [ Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), + Link(url='http://examplesite.com/sample3.html', text=u'sample 3') ]) + if __name__ == "__main__": unittest.main()