diff --git a/scrapy/trunk/scrapy/link/__init__.py b/scrapy/trunk/scrapy/link/__init__.py index 8cd19391c..e59f1d75d 100644 --- a/scrapy/trunk/scrapy/link/__init__.py +++ b/scrapy/trunk/scrapy/link/__init__.py @@ -4,7 +4,7 @@ LinkExtractor provides en efficient way to extract links from pages See documentation in docs/ref/link-extractors.rst """ -from scrapy.utils.python import FixedSGMLParser, unique as unique_list +from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_unicode from scrapy.utils.url import safe_url_string, urljoin_rfc as urljoin class LinkExtractor(FixedSGMLParser): @@ -53,7 +53,7 @@ class LinkExtractor(FixedSGMLParser): for link in links: link.url = urljoin(base_url, link.url) link.url = safe_url_string(link.url, response_encoding) - link.text = link.text.decode(response_encoding) + link.text = str_to_unicode(link.text, response_encoding) ret.append(link) return ret diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/trunk/scrapy/tests/test_link.py index 53528e0ba..9eab45d7b 100644 --- a/scrapy/trunk/scrapy/tests/test_link.py +++ b/scrapy/trunk/scrapy/tests/test_link.py @@ -110,11 +110,6 @@ class RegexLinkExtractorTestCase(unittest.TestCase): self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample2.jpg', text=u'') ]) -# lx = RegexLinkExtractor(restrict_xpaths=('//div[@id="subwrapper"]', )) -# self.assertEqual([link for link in lx.extract_links(self.response)], -# [ Link(url='http://example.com/sample1.html', text=u''), -# Link(url='http://example.com/sample2.html', text=u'sample 2') ]) - def test_extraction_using_single_values(self): '''Test the extractor's behaviour among different situations''' @@ -164,6 +159,30 @@ class RegexLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.matches('http://blah2.com/blah1'), False) self.assertEqual(lx.matches('http://blah2.com/blah2'), False) + def test_restrict_xpaths(self): + lx = RegexLinkExtractor(restrict_xpaths=('//div[@id="subwrapper"]', )) + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2') ]) + + def test_restrict_xpaths_encoding(self): + """Test restrict_xpaths with encodings""" + html = """Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <div class='links'> + <p><a href="/about.html">About us\xa3</a></p> + </div> + <div> + <p><a href="/nofollow.html">This shouldn't be followed</a></p> + </div> + </body></html>""" + response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') + + lx = RegexLinkExtractor(restrict_xpaths="//div[@class='links']") + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.org/about.html', text=u'About us\xa3')]) + + class HTMLImageLinkExtractorTestCase(unittest.TestCase): def setUp(self): base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor')