RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

. Modified LinkExtractors extract_links for being inconsistent. Moved extra parameters to the constructors. 

. Renamed ImageLinkExtractor to HtmlLinkExtractor and moved it to scrapy.contrib.link_extractors.

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40599
This commit is contained in:
elpolilla 2009-01-02 14:06:51 +00:00
parent 2979521b63
commit e5a764cd2a
4 changed files with 100 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
scrapy/trunk/scrapy/contrib/link_extractors.py Normal file
View File

@ -0,0 +1,77 @@
"""
This module provides additional LinkExtractors, apart from the ones in scrapy.link
and scrapy.link.extractors.
"""
import urlparse
from scrapy.link import Link
from scrapy.utils.url import canonicalize_url
from scrapy.utils.python import unicode_to_str
from scrapy.xpath import HtmlXPathSelector
class HTMLImageLinkExtractor(object):
'''HTMLImageLinkExtractor objects are intended to extract image links from HTML pages
given certain xpath locations.
These locations can be passed in a list/tuple either when instanciating the LinkExtractor,
or whenever you call extract_links.
If no locations are specified in any of these places, a default pattern '//img' will be used.
If locations are specified when instanciating the LinkExtractor, and also when calling extract_links,
both locations will be used for that call of extract_links'''
def __init__(self, locations=None, unique=True, canonicalize=True):
self.locations = tuple(locations) if hasattr(locations, '__iter__') else tuple()
self.unique = unique
self.canonicalize = canonicalize
def extract_from_selector(self, selector, parent=None):
ret = []
def _add_link(url_sel, alt_sel=None):
url = url_sel.extract()
alt = alt_sel.extract() if alt_sel else ('', )
if url:
ret.append(Link(unicode_to_str(url[0]), alt[0]))
if selector.xmlNode.type == 'element':
if selector.xmlNode.name == 'img':
_add_link(selector.x('@src'), selector.x('@alt') or selector.x('@title'))
else:
children = selector.x('child::*')
if len(children):
for child in children:
ret.extend(self.extract_from_selector(child, parent=selector))
elif selector.xmlNode.name == 'a' and not parent:
_add_link(selector.x('@href'), selector.x('@title'))
else:
_add_link(selector)
return ret
def extract_links(self, response):
xs = HtmlXPathSelector(response)
base_url = xs.x('//base/@href').extract()
base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url)
links = []
for location in self.locations:
selector_res = xs.x(location)
for selector in selector_res:
links.extend(self.extract_from_selector(selector))
seen, ret = set(), []
for link in links:
link.url = urlparse.urljoin(base_url, link.url)
if self.unique:
if link.url in seen:
continue
else:
seen.add(link.url)
if self.canonicalize:
link.url = canonicalize_url(link.url)
ret.append(link)
return ret
def matches(self, url):
return False

12
scrapy/trunk/scrapy/link/__init__.py
View File

@ -29,15 +29,15 @@ class LinkExtractor(FixedSGMLParser):
* a function which receives an attribute name and returns whether to scan it
"""
def __init__(self, tag="a", attr="href"):
def __init__(self, tag="a", attr="href", unique=False):
FixedSGMLParser.__init__(self)
self.scan_tag = tag if callable(tag) else lambda t: t == tag
self.scan_attr = attr if callable(attr) else lambda a: a == attr
self.current_link = None
def _extract_links(self, response_text, response_url, unique):
self.reset()
self.unique = unique
def _extract_links(self, response_text, response_url):
self.reset()
self.feed(response_text)
self.close()
@ -46,9 +46,9 @@ class LinkExtractor(FixedSGMLParser):
link.url = urljoin(base_url, link.url).strip()
yield link
def extract_links(self, response, unique=False):
def extract_links(self, response):
# wrapper needed to allow to work directly with text
return self._extract_links(response.body.to_string(), response.url, unique)
return self._extract_links(response.body.to_string(), response.url)
def reset(self):
FixedSGMLParser.reset(self)

82
scrapy/trunk/scrapy/link/extractors.py
View File

@ -5,11 +5,9 @@ This module provides some LinkExtractors, which extend that base LinkExtractor
"""
import re
import urlparse
from scrapy.link import LinkExtractor, Link
from scrapy.link import LinkExtractor
from scrapy.utils.url import canonicalize_url, url_is_from_any_domain
from scrapy.utils.python import unicode_to_str
from scrapy.xpath import HtmlXPathSelector
_re_type = type(re.compile("", 0))
@ -42,24 +40,25 @@ class RegexLinkExtractor(LinkExtractor):
"""
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
tags=('a', 'area'), attrs=('href'), canonicalize=True):
tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True):
self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in allow]
self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in deny]
self.allow_domains = set(allow_domains)
self.deny_domains = set(deny_domains)
self.restrict_xpaths = restrict_xpaths
self.canonicalize = canonicalize
self.unique = unique
tag_func = lambda x: x in tags
attr_func = lambda x: x in attrs
LinkExtractor.__init__(self, tag=tag_func, attr=attr_func)
def extract_links(self, response, unique=True):
def extract_links(self, response):
if self.restrict_xpaths:
hxs = HtmlXPathSelector(response)
html_slice = ''.join(''.join(html_fragm for html_fragm in hxs.x(xpath_expr).extract()) for xpath_expr in self.restrict_xpaths)
links = self._extract_links(html_slice, response.url, unique)
links = self._extract_links(html_slice, response.url)
else:
links = LinkExtractor.extract_links(self, response, unique)
links = LinkExtractor.extract_links(self, response)
links = (link for link in links if _is_valid_url(link.url))
@ -86,72 +85,3 @@ class RegexLinkExtractor(LinkExtractor):
allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True]
denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else []
return any(allowed) and not any(denied)
class ImageLinkExtractor(object):
'''ImageLinkExtractor objects are intended to extract image links from HTML pages
given certain xpath locations.
These locations can be passed in a list/tuple either when instanciating the LinkExtractor,
or whenever you call extract_links.
If no locations are specified in any of these places, a default pattern '//img' will be used.
If locations are specified when instanciating the LinkExtractor, and also when calling extract_links,
both locations will be used for that call of extract_links'''
def __init__(self, locations=None):
self.locations = tuple(locations) if hasattr(locations, '__iter__') else tuple()
def extract_from_selector(self, selector, parent=None):
ret = []
def _add_link(url_sel, alt_sel=None):
url = url_sel.extract()
alt = alt_sel.extract() if alt_sel else ('', )
if url:
ret.append(Link(unicode_to_str(url[0]), alt[0]))
if selector.xmlNode.type == 'element':
if selector.xmlNode.name == 'img':
_add_link(selector.x('@src'), selector.x('@alt') or selector.x('@title'))
else:
children = selector.x('child::*')
if len(children):
for child in children:
ret.extend(self.extract_from_selector(child, parent=selector))
elif selector.xmlNode.name == 'a' and not parent:
_add_link(selector.x('@href'), selector.x('@title'))
else:
_add_link(selector)
return ret
def extract_links(self, response, locations=None, unique=True):
xs = HtmlXPathSelector(response)
base_url = xs.x('//base/@href').extract()
base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url)
if hasattr(locations, '__iter__'):
locations = self.locations + tuple(locations)
elif self.locations:
locations = self.locations
else:
locations = ('//img', )
links = []
for location in locations:
selector_res = xs.x(location)
for selector in selector_res:
links.extend(self.extract_from_selector(selector))
seen, ret = set(), []
for link in links:
link.url = urlparse.urljoin(base_url, link.url)
if unique:
if link.url in seen:
continue
else:
seen.add(link.url)
ret.append(link)
return ret
def matches(self, url):
return False

18
scrapy/trunk/scrapy/tests/test_link.py
View File

@ -3,7 +3,8 @@ import unittest
from scrapy.http.response import Response, ResponseBody
from scrapy.link import LinkExtractor, Link
from scrapy.link.extractors import RegexLinkExtractor, ImageLinkExtractor
from scrapy.link.extractors import RegexLinkExtractor
from scrapy.contrib.link_extractors import HTMLImageLinkExtractor
class LinkExtractorTestCase(unittest.TestCase):
def test_basic(self):
@ -66,7 +67,7 @@ class LinkExtractorTestCase(unittest.TestCase):
self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
self.assertEqual(lx.matches('http://blah2.com/blah2'), False)
#class ImageLinkExtractorTestCase(unittest.TestCase):
#class HTMLImageLinkExtractorTestCase(unittest.TestCase):
# def setUp(self):
# body = open(os.path.join(os.path.dirname(__file__), 'sample_data/image_linkextractor.html'), 'r').read()
# self.response = Response(url='http://examplesite.com/index', domain='examplesite.com', body=ResponseBody(body))
@ -79,28 +80,31 @@ class LinkExtractorTestCase(unittest.TestCase):
# def test_extraction(self):
# '''Test the extractor's behaviour among different situations'''
# lx = ImageLinkExtractor()
# links_1 = lx.extract_links(self.response) # using default locations (//img)
# lx = HTMLImageLinkExtractor(locations=('//img', ))
# links_1 = lx.extract_links(self.response)
# self.assertEqual(links_1,
# [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'),
# Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ])
# links_2 = lx.extract_links(self.response, unique=False) # using default locations and unique=False
# lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False)
# links_2 = lx.extract_links(self.response, unique=False)
# self.assertEqual(links_2,
# [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'),
# Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4'),
# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4 repetition') ])
# links_3 = lx.extract_links(self.response, locations=('//div[@id="wrapper"]', ))
# lx = HTMLImageLinkExtractor(locations=('//div[@id="wrapper"]', )
# links_3 = lx.extract_links(self.response)
# self.assertEqual(links_3,
# [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'),
# Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ])
# links_4 = lx.extract_links(self.response, locations=('//a', ))
# lx = HTMLImageLinkExtractor(locations=('//a', )
# links_4 = lx.extract_links(self.response)
# self.assertEqual(links_4,
# [ Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
# Link(url='http://examplesite.com/sample3.html', text=u'sample 3') ])