- Added repr method to Link objects

- Added ImageLinkExtractor and tests

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40592

scrapy/trunk/scrapy/link/__init__.py

@@ -91,3 +91,6 @@ class Link(object):
 
     def __eq__(self, other):
         return self.url == other.url and self.text == other.text
+
+    def __repr__(self):
+        return '<Link url=%r text=%r >' % (self.url, self.text)

scrapy/trunk/scrapy/link/extractors.py

@@ -5,11 +5,13 @@ This module provides some LinkExtractors, which extend that base LinkExtractor
 """
 
 import re
+import urlparse
 
-from scrapy.link import LinkExtractor
+from scrapy.link import LinkExtractor, Link
 from scrapy.utils.url import canonicalize_url, url_is_from_any_domain
 from scrapy.utils.response import new_response_from_xpaths
-from scrapy.utils.misc import dict_updatedefault
+from scrapy.utils.python import unicode_to_str
+from scrapy.xpath.selector import HtmlXPathSelector
 
 _re_type = type(re.compile("", 0))
 
@@ -83,3 +85,63 @@ class RegexLinkExtractor(LinkExtractor):
         allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True]
         denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else []
         return any(allowed) and not any(denied)
+
+class ImageLinkExtractor(object):
+    def __init__(self, locations=None):
+        self.locations = tuple(locations) if hasattr(locations, '__iter__') else tuple()
+
+    def extract_from_selector(self, selector, parent=None):
+        ret = []
+        def _add_link(url_sel, alt_sel=None):
+            url = url_sel.extract()
+            alt = alt_sel.extract() if alt_sel else ('', )
+            if url:
+                ret.append(Link(unicode_to_str(url[0]), alt[0]))
+
+        if selector.xmlNode.type == 'element':
+            if selector.xmlNode.name == 'img':
+                _add_link(selector.x('@src'), selector.x('@alt') or selector.x('@title'))
+            else:
+                children = selector.x('child::*')
+                if len(children):
+                    for child in children:
+                        ret.extend(self.extract_from_selector(child, parent=selector))
+                elif selector.xmlNode.name == 'a' and not parent:
+                    _add_link(selector.x('@href'), selector.x('@title'))
+        else:
+            _add_link(selector)
+
+        return ret
+
+    def extract_links(self, response, locations=None, unique=True):
+        xs = HtmlXPathSelector(response)
+        base_url = xs.x('//base/@href').extract()
+        base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url)
+
+        if hasattr(locations, '__iter__'):
+            locations = self.locations + tuple(locations)
+        elif self.locations:
+            locations = self.locations
+        else:
+            locations = ('//img', )
+
+        links = []
+        for location in locations:
+            selector_res = xs.x(location)
+            for selector in selector_res:
+                links.extend(self.extract_from_selector(selector))
+
+        seen, ret = set(), []
+        for link in links:
+            link.url = urlparse.urljoin(base_url, link.url)
+            if unique:
+                if link.url in seen:
+                    continue
+                else:
+                    seen.add(link.url)
+            ret.append(link)
+
+        return ret
+
+    def matches(self, url):
+        return False

scrapy/trunk/scrapy/tests/test_link.py

@@ -1,8 +1,9 @@
+import os
 import unittest
 
-from scrapy.http import Response
+from scrapy.http.response import Response, ResponseBody
 from scrapy.link import LinkExtractor, Link
-from scrapy.link.extractors import RegexLinkExtractor
+from scrapy.link.extractors import RegexLinkExtractor, ImageLinkExtractor
 
 class LinkExtractorTestCase(unittest.TestCase):
     def test_basic(self):
@@ -63,5 +64,44 @@ class LinkExtractorTestCase(unittest.TestCase):
         self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
         self.assertEqual(lx.matches('http://blah2.com/blah2'), False)
 
+class ImageLinkExtractorTestCase(unittest.TestCase):
+    def setUp(self):
+        body = open(os.path.join(os.path.dirname(__file__), 'sample_data/image_linkextractor.html'), 'r').read()
+        self.response = Response(url='http://examplesite.com/index', domain='examplesite.com', body=ResponseBody(body))
+
+    def test_urls_type(self):
+        '''Test that the resulting urls are regular strings and not a unicode objects'''
+        lx = ImageLinkExtractor()
+        links = lx.extract_links(self.response)
+        self.assertTrue(all(isinstance(link.url, str) for link in links))
+
+    def test_extraction(self):
+        '''Test the extractor's behaviour among different situations'''
+        lx = ImageLinkExtractor()
+
+        links_1 = lx.extract_links(self.response) # using default locations (//img)
+        self.assertEqual(links_1,
+            [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'),
+              Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
+              Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ])
+
+        links_2 = lx.extract_links(self.response, unique=False) # using default locations and unique=False
+        self.assertEqual(links_2,
+            [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'),
+              Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
+              Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4'),
+              Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4 repetition') ])
+
+        links_3 = lx.extract_links(self.response, locations=('//div[@id="wrapper"]', ))
+        self.assertEqual(links_3,
+            [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'),
+              Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
+              Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ])
+
+        links_4 = lx.extract_links(self.response, locations=('//a', ))
+        self.assertEqual(links_4,
+            [ Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
+              Link(url='http://examplesite.com/sample3.html', text=u'sample 3') ])
+
 if __name__ == "__main__":
     unittest.main()