From 86923cc694ca18eff832aa54672d422c1a89ad1d Mon Sep 17 00:00:00 2001 From: elpolilla Date: Thu, 22 Jan 2009 16:39:31 +0000 Subject: [PATCH] Changes in HTMLImageLinkExtractor: . Fixed little bug that triggered IndexErrors in some cases . Added support for receiving selectors instead of just raw xpath expressions . Re-enabled tests --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40759 --- .../trunk/scrapy/contrib/link_extractors.py | 21 ++++-- scrapy/trunk/scrapy/tests/test_link.py | 75 ++++++++++--------- 2 files changed, 52 insertions(+), 44 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/link_extractors.py b/scrapy/trunk/scrapy/contrib/link_extractors.py index 31ae6f61c..5ad5c9b22 100644 --- a/scrapy/trunk/scrapy/contrib/link_extractors.py +++ b/scrapy/trunk/scrapy/contrib/link_extractors.py @@ -1,14 +1,13 @@ """ This module provides additional LinkExtractors, apart from the ones in scrapy.link and scrapy.link.extractors. - """ import urlparse from scrapy.link import Link from scrapy.utils.url import canonicalize_url -from scrapy.utils.python import unicode_to_str -from scrapy.xpath import HtmlXPathSelector +from scrapy.utils.python import unicode_to_str, flatten +from scrapy.xpath.selector import XPathSelectorList, HtmlXPathSelector class HTMLImageLinkExtractor(object): '''HTMLImageLinkExtractor objects are intended to extract image links from HTML pages @@ -21,15 +20,15 @@ class HTMLImageLinkExtractor(object): both locations will be used for that call of extract_links''' def __init__(self, locations=None, unique=True, canonicalize=True): - self.locations = tuple(locations) if hasattr(locations, '__iter__') else tuple() + self.locations = flatten([locations]) self.unique = unique self.canonicalize = canonicalize def extract_from_selector(self, selector, parent=None): ret = [] def _add_link(url_sel, alt_sel=None): - url = url_sel.extract() - alt = alt_sel.extract() if alt_sel else ('', ) + url = flatten([url_sel.extract()]) + alt = flatten([alt_sel.extract()]) if alt_sel else (u'', ) if url: ret.append(Link(unicode_to_str(url[0]), alt[0])) @@ -55,8 +54,14 @@ class HTMLImageLinkExtractor(object): links = [] for location in self.locations: - selector_res = xs.x(location) - for selector in selector_res: + if isinstance(location, basestring): + selectors = xs.x(location) + elif isinstance(location, (XPathSelectorList, HtmlXPathSelector)): + selectors = [location] if isinstance(location, HtmlXPathSelector) else location + else: + continue + + for selector in selectors: links.extend(self.extract_from_selector(selector)) seen, ret = set(), [] diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/trunk/scrapy/tests/test_link.py index e7be4f6f7..f137d218a 100644 --- a/scrapy/trunk/scrapy/tests/test_link.py +++ b/scrapy/trunk/scrapy/tests/test_link.py @@ -142,48 +142,51 @@ class RegexLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.matches('http://blah2.com/blah1'), False) self.assertEqual(lx.matches('http://blah2.com/blah2'), False) -#class HTMLImageLinkExtractorTestCase(unittest.TestCase): -# def setUp(self): -# base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor') -# body = open(os.path.join(base_path 'image_linkextractor.html'), 'r').read() -# self.response = Response(url='http://example.com/index', body=body) +class HTMLImageLinkExtractorTestCase(unittest.TestCase): + def setUp(self): + base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor') + body = open(os.path.join(base_path, 'image_linkextractor.html'), 'r').read() + self.response = Response(url='http://example.com/index', body=body) -# def test_urls_type(self): -# '''Test that the resulting urls are regular strings and not a unicode objects''' -# lx = ImageLinkExtractor() -# links = lx.extract_links(self.response) -# self.assertTrue(all(isinstance(link.url, str) for link in links)) + def tearDown(self): + del self.response -# def test_extraction(self): -# '''Test the extractor's behaviour among different situations''' + def test_urls_type(self): + '''Test that the resulting urls are regular strings and not a unicode objects''' + lx = HTMLImageLinkExtractor() + links = lx.extract_links(self.response) + self.assertTrue(all(isinstance(link.url, str) for link in links)) -# lx = HTMLImageLinkExtractor(locations=('//img', )) -# links_1 = lx.extract_links(self.response) -# self.assertEqual(links_1, -# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), -# Link(url='http://example.com/sample2.jpg', text=u'sample 2'), -# Link(url='http://example.com/sample4.jpg', text=u'sample 4') ]) + def test_extraction(self): + '''Test the extractor's behaviour among different situations''' -# lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False) -# links_2 = lx.extract_links(self.response, unique=False) -# self.assertEqual(links_2, -# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), -# Link(url='http://example.com/sample2.jpg', text=u'sample 2'), -# Link(url='http://example.com/sample4.jpg', text=u'sample 4'), -# Link(url='http://example.com/sample4.jpg', text=u'sample 4 repetition') ]) + lx = HTMLImageLinkExtractor(locations=('//img', )) + links_1 = lx.extract_links(self.response) + self.assertEqual(links_1, + [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), + Link(url='http://example.com/sample2.jpg', text=u'sample 2'), + Link(url='http://example.com/sample4.jpg', text=u'sample 4') ]) -# lx = HTMLImageLinkExtractor(locations=('//div[@id="wrapper"]', ) -# links_3 = lx.extract_links(self.response) -# self.assertEqual(links_3, -# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), -# Link(url='http://example.com/sample2.jpg', text=u'sample 2'), -# Link(url='http://example.com/sample4.jpg', text=u'sample 4') ]) + lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False) + links_2 = lx.extract_links(self.response) + self.assertEqual(links_2, + [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), + Link(url='http://example.com/sample2.jpg', text=u'sample 2'), + Link(url='http://example.com/sample4.jpg', text=u'sample 4'), + Link(url='http://example.com/sample4.jpg', text=u'sample 4 repetition') ]) -# lx = HTMLImageLinkExtractor(locations=('//a', ) -# links_4 = lx.extract_links(self.response) -# self.assertEqual(links_4, -# [ Link(url='http://example.com/sample2.jpg', text=u'sample 2'), -# Link(url='http://example.com/sample3.html', text=u'sample 3') ]) + lx = HTMLImageLinkExtractor(locations=('//div[@id="wrapper"]', )) + links_3 = lx.extract_links(self.response) + self.assertEqual(links_3, + [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), + Link(url='http://example.com/sample2.jpg', text=u'sample 2'), + Link(url='http://example.com/sample4.jpg', text=u'sample 4') ]) + + lx = HTMLImageLinkExtractor(locations=('//a', )) + links_4 = lx.extract_links(self.response) + self.assertEqual(links_4, + [ Link(url='http://example.com/sample2.jpg', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3') ]) if __name__ == "__main__": unittest.main()