mirror of https://github.com/scrapy/scrapy.git
. Added tests for RegexLinkExtractor
. Modified LinkExtractor in order to percent-encode urls using the response encoding . Improved LinkExtractors processing of unique links --HG-- rename : scrapy/trunk/scrapy/tests/sample_data/image_linkextractor.html => scrapy/trunk/scrapy/tests/sample_data/link_extractor/image_linkextractor.html extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40609
This commit is contained in:
elpolilla
parent
c791155aac
commit
63662ba6ff
|
|
@ -3,21 +3,22 @@ LinkExtractor provides en efficient way to extract links from pages
|
|||
"""
|
||||
|
||||
from scrapy.utils.python import FixedSGMLParser
|
||||
from scrapy.utils.url import urljoin_rfc as urljoin
|
||||
from scrapy.utils.url import safe_url_string, urljoin_rfc as urljoin
|
||||
|
||||
class LinkExtractor(FixedSGMLParser):
|
||||
"""LinkExtractor are used to extract links from web pages. They are
|
||||
instantiated and later "applied" to a Response using the extract_links
|
||||
method which must receive a Response object and return a dict whoose keys
|
||||
are the (absolute) urls to follow, and its values any arbitrary data. In
|
||||
this case the values are the text of the hyperlink.
|
||||
method which must receive a Response object and return a list of Link objects
|
||||
containing the (absolute) urls to follow, and the links texts.
|
||||
|
||||
This is the base LinkExtractor class that provides enough basic
|
||||
functionality for extracting links to follow, but you could override this
|
||||
class or create a new one if you need some additional functionality. The
|
||||
only requisite is that the new (or overrided) class must provide a
|
||||
extract_links method that receives a Response and returns a dict with the
|
||||
links to follow as its keys.
|
||||
extract_links method that receives a Response and returns a list of Link objects.
|
||||
|
||||
This LinkExtractor always returns percent-encoded URLs, using the detected encoding
|
||||
from the response.
|
||||
|
||||
The constructor arguments are:
|
||||
|
||||
|
|
@ -36,22 +37,38 @@ class LinkExtractor(FixedSGMLParser):
|
|||
self.current_link = None
|
||||
self.unique = unique
|
||||
|
||||
def _extract_links(self, response_text, response_url):
|
||||
def _extract_links(self, response_text, response_url, response_encoding):
|
||||
self.reset()
|
||||
self.feed(response_text)
|
||||
self.close()
|
||||
|
||||
base_url = self.base_url if self.base_url else response_url
|
||||
links = self.links
|
||||
if self.unique:
|
||||
seen = set()
|
||||
def _seen(url):
|
||||
if url in seen:
|
||||
return True
|
||||
else:
|
||||
seen.add(url)
|
||||
return False
|
||||
|
||||
links = []
|
||||
for link in self.links:
|
||||
links = [link for link in links if not _seen(link.url)]
|
||||
|
||||
ret = []
|
||||
base_url = self.base_url if self.base_url else response_url
|
||||
for link in links:
|
||||
link.url = urljoin(base_url, link.url).strip()
|
||||
links.append(link)
|
||||
return links
|
||||
link.url = safe_url_string(link.url, response_encoding)
|
||||
link.text = link.text.decode(response_encoding)
|
||||
ret.append(link)
|
||||
|
||||
return ret
|
||||
|
||||
def extract_links(self, response):
|
||||
# wrapper needed to allow to work directly with text
|
||||
return self._extract_links(response.body.to_string(), response.url)
|
||||
return self._extract_links(response.body.to_string(),
|
||||
response.url,
|
||||
response.body.get_real_encoding())
|
||||
|
||||
def reset(self):
|
||||
FixedSGMLParser.reset(self)
|
||||
|
|
@ -64,10 +81,9 @@ class LinkExtractor(FixedSGMLParser):
|
|||
if self.scan_tag(tag):
|
||||
for attr, value in attrs:
|
||||
if self.scan_attr(attr):
|
||||
if not self.unique or not value in [link.url for link in self.links]:
|
||||
link = Link(url=value)
|
||||
self.links.append(link)
|
||||
self.current_link = link
|
||||
link = Link(url=value)
|
||||
self.links.append(link)
|
||||
self.current_link = link
|
||||
|
||||
def unknown_endtag(self, tag):
|
||||
self.current_link = None
|
||||
|
|
|
|||
|
|
@ -47,10 +47,9 @@ class RegexLinkExtractor(LinkExtractor):
|
|||
self.deny_domains = set(deny_domains)
|
||||
self.restrict_xpaths = restrict_xpaths
|
||||
self.canonicalize = canonicalize
|
||||
self.unique = unique
|
||||
tag_func = lambda x: x in tags
|
||||
attr_func = lambda x: x in attrs
|
||||
LinkExtractor.__init__(self, tag=tag_func, attr=attr_func)
|
||||
LinkExtractor.__init__(self, tag=tag_func, attr=attr_func, unique=unique)
|
||||
|
||||
def extract_links(self, response):
|
||||
if self.restrict_xpaths:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<html>
|
||||
<head>
|
||||
<base href='http://examplesite.com' />
|
||||
<title>Sample page with image links for testing ImageLinkExtractor</title>
|
||||
<base href='http://example.com' />
|
||||
<title>Sample page with image links for testing HTMLImageLinkExtractor</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='wrapper'>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=latin-1">
|
||||
<base href='http://example.com' />
|
||||
<title>Sample page with links for testing RegexLinkExtractor</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='wrapper'>
|
||||
<div id='subwrapper'>
|
||||
<a href='sample_ñ.html'><img src='sample2.jpg'/></a>
|
||||
</div>
|
||||
<a href='sample_á.html' title='sample á'>sample á text</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
<html>
|
||||
<head>
|
||||
<base href='http://example.com' />
|
||||
<title>Sample page without encoding for testing LinkExtractor</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='wrapper'>
|
||||
<div id='subwrapper'>
|
||||
<a href='sample_ñ.html'><img src='sample2.jpg'/></a>
|
||||
</div>
|
||||
<a href='sample_€.html' title='sample €'>sample € text</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
<html>
|
||||
<head>
|
||||
<base href='http://example.com' />
|
||||
<title>Sample page with links for testing RegexLinkExtractor</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='wrapper'>
|
||||
<div id='subwrapper'>
|
||||
<area href='sample1.html'>sample 1</area>
|
||||
<a href='sample2.html'><img src='sample2.jpg'/></a>
|
||||
</div>
|
||||
<a href='sample3.html' title='sample 3'>sample 3 text</a>
|
||||
<a href='sample3.html'>sample 3 repetition</a>
|
||||
<a href='http://www.google.com/something'></a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -34,6 +34,27 @@ class LinkExtractorTestCase(unittest.TestCase):
|
|||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
|
||||
|
||||
def test_extraction_encoding(self):
|
||||
base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor')
|
||||
body = open(os.path.join(base_path, 'linkextractor_noenc.html'), 'r').read()
|
||||
response_utf8 = Response(url='http://example.com/utf8', domain='example.com', body=ResponseBody(body), headers={'Content-Type': ['text/html; charset=utf-8']})
|
||||
response_noenc = Response(url='http://example.com/noenc', domain='example.com', body=ResponseBody(body))
|
||||
body = open(os.path.join(base_path, 'linkextractor_latin1.html'), 'r').read()
|
||||
response_latin1 = Response(url='http://example.com/latin1', domain='example.com', body=ResponseBody(body))
|
||||
|
||||
lx = LinkExtractor()
|
||||
self.assertEqual(lx.extract_links(response_utf8),
|
||||
[ Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')) ])
|
||||
|
||||
self.assertEqual(lx.extract_links(response_noenc),
|
||||
[ Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')) ])
|
||||
|
||||
self.assertEqual(lx.extract_links(response_latin1),
|
||||
[ Link(url='http://example.com/sample_%F1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')) ])
|
||||
|
||||
def test_matches(self):
|
||||
url1 = 'http://lotsofstuff.com/stuff1/index'
|
||||
url2 = 'http://evenmorestuff.com/uglystuff/index'
|
||||
|
|
@ -42,6 +63,57 @@ class LinkExtractorTestCase(unittest.TestCase):
|
|||
self.assertEqual(lx.matches(url1), True)
|
||||
self.assertEqual(lx.matches(url2), True)
|
||||
|
||||
class RegexLinkExtractorTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor')
|
||||
body = open(os.path.join(base_path, 'regex_linkextractor.html'), 'r').read()
|
||||
self.response = Response(url='http://example.com/index', domain='example.com', body=ResponseBody(body))
|
||||
|
||||
def test_urls_type(self):
|
||||
'''Test that the resulting urls are regular strings and not a unicode objects'''
|
||||
lx = RegexLinkExtractor()
|
||||
self.assertTrue(all(isinstance(link.url, str) for link in lx.extract_links(self.response)))
|
||||
|
||||
def test_extraction(self):
|
||||
'''Test the extractor's behaviour among different situations'''
|
||||
|
||||
lx = RegexLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text='sample 1'),
|
||||
Link(url='http://example.com/sample2.html', text=''),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text='') ])
|
||||
|
||||
lx = RegexLinkExtractor(allow=('sample', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text='sample 1'),
|
||||
Link(url='http://example.com/sample2.html', text=''),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text') ])
|
||||
|
||||
lx = RegexLinkExtractor(allow=('sample', ), unique=False)
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text='sample 1'),
|
||||
Link(url='http://example.com/sample2.html', text=''),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 repetition') ])
|
||||
|
||||
lx = RegexLinkExtractor(allow=('sample', ), deny=('3', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text='sample 1'),
|
||||
Link(url='http://example.com/sample2.html', text='') ])
|
||||
|
||||
lx = RegexLinkExtractor(allow_domains=('google.com', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://www.google.com/something', text='') ])
|
||||
|
||||
lx = RegexLinkExtractor(tags=('img', ), attrs=('src', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample2.jpg', text='') ])
|
||||
|
||||
def test_matches(self):
|
||||
url1 = 'http://lotsofstuff.com/stuff1/index'
|
||||
url2 = 'http://evenmorestuff.com/uglystuff/index'
|
||||
|
||||
lx = RegexLinkExtractor(allow=(r'stuff1', ))
|
||||
self.assertEqual(lx.matches(url1), True)
|
||||
self.assertEqual(lx.matches(url2), False)
|
||||
|
|
@ -67,8 +139,9 @@ class LinkExtractorTestCase(unittest.TestCase):
|
|||
|
||||
#class HTMLImageLinkExtractorTestCase(unittest.TestCase):
|
||||
# def setUp(self):
|
||||
# body = open(os.path.join(os.path.dirname(__file__), 'sample_data/image_linkextractor.html'), 'r').read()
|
||||
# self.response = Response(url='http://examplesite.com/index', domain='examplesite.com', body=ResponseBody(body))
|
||||
# base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor')
|
||||
# body = open(os.path.join(base_path 'image_linkextractor.html'), 'r').read()
|
||||
# self.response = Response(url='http://example.com/index', domain='example.com', body=ResponseBody(body))
|
||||
|
||||
# def test_urls_type(self):
|
||||
# '''Test that the resulting urls are regular strings and not a unicode objects'''
|
||||
|
|
@ -82,30 +155,30 @@ class LinkExtractorTestCase(unittest.TestCase):
|
|||
# lx = HTMLImageLinkExtractor(locations=('//img', ))
|
||||
# links_1 = lx.extract_links(self.response)
|
||||
# self.assertEqual(links_1,
|
||||
# [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'),
|
||||
# Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
|
||||
# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ])
|
||||
# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
|
||||
# Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
|
||||
# Link(url='http://example.com/sample4.jpg', text=u'sample 4') ])
|
||||
|
||||
# lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False)
|
||||
# links_2 = lx.extract_links(self.response, unique=False)
|
||||
# self.assertEqual(links_2,
|
||||
# [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'),
|
||||
# Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
|
||||
# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4'),
|
||||
# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4 repetition') ])
|
||||
# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
|
||||
# Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
|
||||
# Link(url='http://example.com/sample4.jpg', text=u'sample 4'),
|
||||
# Link(url='http://example.com/sample4.jpg', text=u'sample 4 repetition') ])
|
||||
|
||||
# lx = HTMLImageLinkExtractor(locations=('//div[@id="wrapper"]', )
|
||||
# links_3 = lx.extract_links(self.response)
|
||||
# self.assertEqual(links_3,
|
||||
# [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'),
|
||||
# Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
|
||||
# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ])
|
||||
# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'),
|
||||
# Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
|
||||
# Link(url='http://example.com/sample4.jpg', text=u'sample 4') ])
|
||||
|
||||
# lx = HTMLImageLinkExtractor(locations=('//a', )
|
||||
# links_4 = lx.extract_links(self.response)
|
||||
# self.assertEqual(links_4,
|
||||
# [ Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'),
|
||||
# Link(url='http://examplesite.com/sample3.html', text=u'sample 3') ])
|
||||
# [ Link(url='http://example.com/sample2.jpg', text=u'sample 2'),
|
||||
# Link(url='http://example.com/sample3.html', text=u'sample 3') ])
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue