RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixed encoding problem in RegexLinkExtractor when extracting links using restrict_xpaths, and added unittests. also enabled previously disabled restrict_xpaths tests. 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401100
This commit is contained in:
Pablo Hoffman 2009-04-29 19:04:27 +00:00
parent e5892050bd
commit 239b056ddd
2 changed files with 26 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/trunk/scrapy/link/__init__.py
View File

@ -4,7 +4,7 @@ LinkExtractor provides en efficient way to extract links from pages
See documentation in docs/ref/link-extractors.rst
"""
from scrapy.utils.python import FixedSGMLParser, unique as unique_list
from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_unicode
from scrapy.utils.url import safe_url_string, urljoin_rfc as urljoin
class LinkExtractor(FixedSGMLParser):
@ -53,7 +53,7 @@ class LinkExtractor(FixedSGMLParser):
for link in links:
link.url = urljoin(base_url, link.url)
link.url = safe_url_string(link.url, response_encoding)
link.text = link.text.decode(response_encoding)
link.text = str_to_unicode(link.text, response_encoding)
ret.append(link)
return ret

29
scrapy/trunk/scrapy/tests/test_link.py
View File

@ -110,11 +110,6 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
self.assertEqual([link for link in lx.extract_links(self.response)],
[ Link(url='http://example.com/sample2.jpg', text=u'') ])
# lx = RegexLinkExtractor(restrict_xpaths=('//div[@id="subwrapper"]', ))
# self.assertEqual([link for link in lx.extract_links(self.response)],
# [ Link(url='http://example.com/sample1.html', text=u''),
# Link(url='http://example.com/sample2.html', text=u'sample 2') ])
def test_extraction_using_single_values(self):
'''Test the extractor's behaviour among different situations'''
@ -164,6 +159,30 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
self.assertEqual(lx.matches('http://blah2.com/blah2'), False)
def test_restrict_xpaths(self):
lx = RegexLinkExtractor(restrict_xpaths=('//div[@id="subwrapper"]', ))
self.assertEqual([link for link in lx.extract_links(self.response)],
[ Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2') ])
def test_restrict_xpaths_encoding(self):
"""Test restrict_xpaths with encodings"""
html = """<html><head><title>Page title<title>
<body><p><a href="item/12.html">Item 12</a></p>
<div class='links'>
<p><a href="/about.html">About us\xa3</a></p>
</div>
<div>
<p><a href="/nofollow.html">This shouldn't be followed</a></p>
</div>
</body></html>"""
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
lx = RegexLinkExtractor(restrict_xpaths="//div[@class='links']")
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/about.html', text=u'About us\xa3')])
class HTMLImageLinkExtractorTestCase(unittest.TestCase):
def setUp(self):
base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor')