diff --git a/scrapy/trunk/scrapy/contrib/link_extractors.py b/scrapy/trunk/scrapy/contrib/link_extractors.py new file mode 100644 index 000000000..31ae6f61c --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/link_extractors.py @@ -0,0 +1,77 @@ +""" +This module provides additional LinkExtractors, apart from the ones in scrapy.link +and scrapy.link.extractors. + +""" +import urlparse + +from scrapy.link import Link +from scrapy.utils.url import canonicalize_url +from scrapy.utils.python import unicode_to_str +from scrapy.xpath import HtmlXPathSelector + +class HTMLImageLinkExtractor(object): + '''HTMLImageLinkExtractor objects are intended to extract image links from HTML pages + given certain xpath locations. + + These locations can be passed in a list/tuple either when instanciating the LinkExtractor, + or whenever you call extract_links. + If no locations are specified in any of these places, a default pattern '//img' will be used. + If locations are specified when instanciating the LinkExtractor, and also when calling extract_links, + both locations will be used for that call of extract_links''' + + def __init__(self, locations=None, unique=True, canonicalize=True): + self.locations = tuple(locations) if hasattr(locations, '__iter__') else tuple() + self.unique = unique + self.canonicalize = canonicalize + + def extract_from_selector(self, selector, parent=None): + ret = [] + def _add_link(url_sel, alt_sel=None): + url = url_sel.extract() + alt = alt_sel.extract() if alt_sel else ('', ) + if url: + ret.append(Link(unicode_to_str(url[0]), alt[0])) + + if selector.xmlNode.type == 'element': + if selector.xmlNode.name == 'img': + _add_link(selector.x('@src'), selector.x('@alt') or selector.x('@title')) + else: + children = selector.x('child::*') + if len(children): + for child in children: + ret.extend(self.extract_from_selector(child, parent=selector)) + elif selector.xmlNode.name == 'a' and not parent: + _add_link(selector.x('@href'), selector.x('@title')) + else: + _add_link(selector) + + return ret + + def extract_links(self, response): + xs = HtmlXPathSelector(response) + base_url = xs.x('//base/@href').extract() + base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url) + + links = [] + for location in self.locations: + selector_res = xs.x(location) + for selector in selector_res: + links.extend(self.extract_from_selector(selector)) + + seen, ret = set(), [] + for link in links: + link.url = urlparse.urljoin(base_url, link.url) + if self.unique: + if link.url in seen: + continue + else: + seen.add(link.url) + if self.canonicalize: + link.url = canonicalize_url(link.url) + ret.append(link) + + return ret + + def matches(self, url): + return False diff --git a/scrapy/trunk/scrapy/link/__init__.py b/scrapy/trunk/scrapy/link/__init__.py index 5e15fb6fb..c52994995 100644 --- a/scrapy/trunk/scrapy/link/__init__.py +++ b/scrapy/trunk/scrapy/link/__init__.py @@ -29,15 +29,15 @@ class LinkExtractor(FixedSGMLParser): * a function which receives an attribute name and returns whether to scan it """ - def __init__(self, tag="a", attr="href"): + def __init__(self, tag="a", attr="href", unique=False): FixedSGMLParser.__init__(self) self.scan_tag = tag if callable(tag) else lambda t: t == tag self.scan_attr = attr if callable(attr) else lambda a: a == attr self.current_link = None - - def _extract_links(self, response_text, response_url, unique): - self.reset() self.unique = unique + + def _extract_links(self, response_text, response_url): + self.reset() self.feed(response_text) self.close() @@ -46,9 +46,9 @@ class LinkExtractor(FixedSGMLParser): link.url = urljoin(base_url, link.url).strip() yield link - def extract_links(self, response, unique=False): + def extract_links(self, response): # wrapper needed to allow to work directly with text - return self._extract_links(response.body.to_string(), response.url, unique) + return self._extract_links(response.body.to_string(), response.url) def reset(self): FixedSGMLParser.reset(self) diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/trunk/scrapy/link/extractors.py index d86aa3bf5..d2a31676c 100644 --- a/scrapy/trunk/scrapy/link/extractors.py +++ b/scrapy/trunk/scrapy/link/extractors.py @@ -5,11 +5,9 @@ This module provides some LinkExtractors, which extend that base LinkExtractor """ import re -import urlparse -from scrapy.link import LinkExtractor, Link +from scrapy.link import LinkExtractor from scrapy.utils.url import canonicalize_url, url_is_from_any_domain -from scrapy.utils.python import unicode_to_str from scrapy.xpath import HtmlXPathSelector _re_type = type(re.compile("", 0)) @@ -42,24 +40,25 @@ class RegexLinkExtractor(LinkExtractor): """ def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href'), canonicalize=True): + tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True): self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in allow] self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in deny] self.allow_domains = set(allow_domains) self.deny_domains = set(deny_domains) self.restrict_xpaths = restrict_xpaths self.canonicalize = canonicalize + self.unique = unique tag_func = lambda x: x in tags attr_func = lambda x: x in attrs LinkExtractor.__init__(self, tag=tag_func, attr=attr_func) - def extract_links(self, response, unique=True): + def extract_links(self, response): if self.restrict_xpaths: hxs = HtmlXPathSelector(response) html_slice = ''.join(''.join(html_fragm for html_fragm in hxs.x(xpath_expr).extract()) for xpath_expr in self.restrict_xpaths) - links = self._extract_links(html_slice, response.url, unique) + links = self._extract_links(html_slice, response.url) else: - links = LinkExtractor.extract_links(self, response, unique) + links = LinkExtractor.extract_links(self, response) links = (link for link in links if _is_valid_url(link.url)) @@ -86,72 +85,3 @@ class RegexLinkExtractor(LinkExtractor): allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True] denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else [] return any(allowed) and not any(denied) - -class ImageLinkExtractor(object): - '''ImageLinkExtractor objects are intended to extract image links from HTML pages - given certain xpath locations. - - These locations can be passed in a list/tuple either when instanciating the LinkExtractor, - or whenever you call extract_links. - If no locations are specified in any of these places, a default pattern '//img' will be used. - If locations are specified when instanciating the LinkExtractor, and also when calling extract_links, - both locations will be used for that call of extract_links''' - - def __init__(self, locations=None): - self.locations = tuple(locations) if hasattr(locations, '__iter__') else tuple() - - def extract_from_selector(self, selector, parent=None): - ret = [] - def _add_link(url_sel, alt_sel=None): - url = url_sel.extract() - alt = alt_sel.extract() if alt_sel else ('', ) - if url: - ret.append(Link(unicode_to_str(url[0]), alt[0])) - - if selector.xmlNode.type == 'element': - if selector.xmlNode.name == 'img': - _add_link(selector.x('@src'), selector.x('@alt') or selector.x('@title')) - else: - children = selector.x('child::*') - if len(children): - for child in children: - ret.extend(self.extract_from_selector(child, parent=selector)) - elif selector.xmlNode.name == 'a' and not parent: - _add_link(selector.x('@href'), selector.x('@title')) - else: - _add_link(selector) - - return ret - - def extract_links(self, response, locations=None, unique=True): - xs = HtmlXPathSelector(response) - base_url = xs.x('//base/@href').extract() - base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url) - - if hasattr(locations, '__iter__'): - locations = self.locations + tuple(locations) - elif self.locations: - locations = self.locations - else: - locations = ('//img', ) - - links = [] - for location in locations: - selector_res = xs.x(location) - for selector in selector_res: - links.extend(self.extract_from_selector(selector)) - - seen, ret = set(), [] - for link in links: - link.url = urlparse.urljoin(base_url, link.url) - if unique: - if link.url in seen: - continue - else: - seen.add(link.url) - ret.append(link) - - return ret - - def matches(self, url): - return False diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/trunk/scrapy/tests/test_link.py index e413dc8cd..0ebd51733 100644 --- a/scrapy/trunk/scrapy/tests/test_link.py +++ b/scrapy/trunk/scrapy/tests/test_link.py @@ -3,7 +3,8 @@ import unittest from scrapy.http.response import Response, ResponseBody from scrapy.link import LinkExtractor, Link -from scrapy.link.extractors import RegexLinkExtractor, ImageLinkExtractor +from scrapy.link.extractors import RegexLinkExtractor +from scrapy.contrib.link_extractors import HTMLImageLinkExtractor class LinkExtractorTestCase(unittest.TestCase): def test_basic(self): @@ -66,7 +67,7 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.matches('http://blah2.com/blah1'), False) self.assertEqual(lx.matches('http://blah2.com/blah2'), False) -#class ImageLinkExtractorTestCase(unittest.TestCase): +#class HTMLImageLinkExtractorTestCase(unittest.TestCase): # def setUp(self): # body = open(os.path.join(os.path.dirname(__file__), 'sample_data/image_linkextractor.html'), 'r').read() # self.response = Response(url='http://examplesite.com/index', domain='examplesite.com', body=ResponseBody(body)) @@ -79,28 +80,31 @@ class LinkExtractorTestCase(unittest.TestCase): # def test_extraction(self): # '''Test the extractor's behaviour among different situations''' -# lx = ImageLinkExtractor() -# links_1 = lx.extract_links(self.response) # using default locations (//img) +# lx = HTMLImageLinkExtractor(locations=('//img', )) +# links_1 = lx.extract_links(self.response) # self.assertEqual(links_1, # [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'), # Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), # Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ]) -# links_2 = lx.extract_links(self.response, unique=False) # using default locations and unique=False +# lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False) +# links_2 = lx.extract_links(self.response, unique=False) # self.assertEqual(links_2, # [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'), # Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), # Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4'), # Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4 repetition') ]) -# links_3 = lx.extract_links(self.response, locations=('//div[@id="wrapper"]', )) +# lx = HTMLImageLinkExtractor(locations=('//div[@id="wrapper"]', ) +# links_3 = lx.extract_links(self.response) # self.assertEqual(links_3, # [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'), # Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), # Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ]) -# links_4 = lx.extract_links(self.response, locations=('//a', )) +# lx = HTMLImageLinkExtractor(locations=('//a', ) +# links_4 = lx.extract_links(self.response) # self.assertEqual(links_4, # [ Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), # Link(url='http://examplesite.com/sample3.html', text=u'sample 3') ])