RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Changes in HTMLImageLinkExtractor: 

. Fixed little bug that triggered IndexErrors in some cases
. Added support for receiving selectors instead of just raw xpath expressions
. Re-enabled tests

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40759
This commit is contained in:
elpolilla 2009-01-22 16:39:31 +00:00
parent eebc070fe4
commit 86923cc694
2 changed files with 52 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
scrapy/trunk/scrapy/contrib/link_extractors.py
View File

@ -1,14 +1,13 @@
"""
This module provides additional LinkExtractors, apart from the ones in scrapy.link
and scrapy.link.extractors.
"""
import urlparse
from scrapy.link import Link
from scrapy.utils.url import canonicalize_url
from scrapy.utils.python import unicode_to_str
from scrapy.xpath import HtmlXPathSelector
from scrapy.utils.python import unicode_to_str, flatten
from scrapy.xpath.selector import XPathSelectorList, HtmlXPathSelector
class HTMLImageLinkExtractor(object):
'''HTMLImageLinkExtractor objects are intended to extract image links from HTML pages
@ -21,15 +20,15 @@ class HTMLImageLinkExtractor(object):
both locations will be used for that call of extract_links'''
def __init__(self, locations=None, unique=True, canonicalize=True):
self.locations = tuple(locations) if hasattr(locations, '__iter__') else tuple()
self.locations = flatten([locations])
self.unique = unique
self.canonicalize = canonicalize
def extract_from_selector(self, selector, parent=None):
ret = []
def _add_link(url_sel, alt_sel=None):
url = url_sel.extract()
alt = alt_sel.extract() if alt_sel else ('', )
url = flatten([url_sel.extract()])
alt = flatten([alt_sel.extract()]) if alt_sel else (u'', )
if url:
ret.append(Link(unicode_to_str(url[0]), alt[0]))
@ -55,8 +54,14 @@ class HTMLImageLinkExtractor(object):
links = []
for location in self.locations:
selector_res = xs.x(location)
for selector in selector_res:
if isinstance(location, basestring):
selectors = xs.x(location)
elif isinstance(location, (XPathSelectorList, HtmlXPathSelector)):
selectors = [location] if isinstance(location, HtmlXPathSelector) else location
else:
continue
for selector in selectors:
links.extend(self.extract_from_selector(selector))
seen, ret = set(), []

75
scrapy/trunk/scrapy/tests/test_link.py
View File

@ -142,48 +142,51 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
self.assertEqual(lx.matches('http://blah2.com/blah2'), False)
#class HTMLImageLinkExtractorTestCase(unittest.TestCase):
# def setUp(self):
# base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor')
# body = open(os.path.join(base_path 'image_linkextractor.html'), 'r').read()
# self.response = Response(url='http://example.com/index', body=body)
class HTMLImageLinkExtractorTestCase(unittest.TestCase):
def setUp(self):
base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor')
body = open(os.path.join(base_path, 'image_linkextractor.html'), 'r').read()
self.response = Response(url='http://example.com/index', body=body)
# def test_urls_type(self):
# '''Test that the resulting urls are regular strings and not a unicode objects'''
# lx = ImageLinkExtractor()
# links = lx.extract_links(self.response)
# self.assertTrue(all(isinstance(link.url, str) for link in links))
def tearDown(self):
del self.response
# def test_extraction(self):
# '''Test the extractor's behaviour among different situations'''
def test_urls_type(self):
'''Test that the resulting urls are regular strings and not a unicode objects'''
lx = HTMLImageLinkExtractor()
links = lx.extract_links(self.response)
self.assertTrue(all(isinstance(link.url, str) for link in links))
# lx = HTMLImageLinkExtractor(locations=('//img', ))
# links_1 = lx.extract_links(self.response)
# self.assertEqual(links_1,
# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
# Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
# Link(url='http://example.com/sample4.jpg', text=u'sample 4') ])
def test_extraction(self):
'''Test the extractor's behaviour among different situations'''
# lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False)
# links_2 = lx.extract_links(self.response, unique=False)
# self.assertEqual(links_2,
# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
# Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
# Link(url='http://example.com/sample4.jpg', text=u'sample 4'),
# Link(url='http://example.com/sample4.jpg', text=u'sample 4 repetition') ])
lx = HTMLImageLinkExtractor(locations=('//img', ))
links_1 = lx.extract_links(self.response)
self.assertEqual(links_1,
[ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
Link(url='http://example.com/sample4.jpg', text=u'sample 4') ])
# lx = HTMLImageLinkExtractor(locations=('//div[@id="wrapper"]', )
# links_3 = lx.extract_links(self.response)
# self.assertEqual(links_3,
# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
# Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
# Link(url='http://example.com/sample4.jpg', text=u'sample 4') ])
lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False)
links_2 = lx.extract_links(self.response)
self.assertEqual(links_2,
[ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
Link(url='http://example.com/sample4.jpg', text=u'sample 4'),
Link(url='http://example.com/sample4.jpg', text=u'sample 4 repetition') ])
# lx = HTMLImageLinkExtractor(locations=('//a', )
# links_4 = lx.extract_links(self.response)
# self.assertEqual(links_4,
# [ Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
# Link(url='http://example.com/sample3.html', text=u'sample 3') ])
lx = HTMLImageLinkExtractor(locations=('//div[@id="wrapper"]', ))
links_3 = lx.extract_links(self.response)
self.assertEqual(links_3,
[ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
Link(url='http://example.com/sample4.jpg', text=u'sample 4') ])
lx = HTMLImageLinkExtractor(locations=('//a', ))
links_4 = lx.extract_links(self.response)
self.assertEqual(links_4,
[ Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3') ])
if __name__ == "__main__":
unittest.main()