diff --git a/scrapy/contrib/linkextractors/lxmlparser.py b/scrapy/contrib/linkextractors/lxmlparser.py index 27cd0697a..1bf35501d 100644 --- a/scrapy/contrib/linkextractors/lxmlparser.py +++ b/scrapy/contrib/linkextractors/lxmlparser.py @@ -33,7 +33,7 @@ class LxmlLinkExtractor(object): for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) - link.text = str_to_unicode(link.text, response_encoding) + link.text = str_to_unicode(link.text, response_encoding, errors='replace') ret.append(link) return ret diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 932b9eba0..8fbcb35e7 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -33,7 +33,7 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): for link in self.links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) - link.text = str_to_unicode(link.text, response_encoding) + link.text = str_to_unicode(link.text, response_encoding, errors='replace') ret.append(link) return ret diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 9727f34ad..0a4f23814 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -51,6 +51,13 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(response), [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')]) + def test_link_text_wrong_encoding(self): + html = """

Wrong: \xed

""" + response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8') + lx = BaseSgmlLinkExtractor() + self.assertEqual(lx.extract_links(response), + [Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd')]) + def test_extraction_encoding(self): body = get_testdata('link_extractor', 'linkextractor_noenc.html') response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']}) diff --git a/scrapy/tests/test_utils_python.py b/scrapy/tests/test_utils_python.py index 3c4c9fd6f..238c36f1d 100644 --- a/scrapy/tests/test_utils_python.py +++ b/scrapy/tests/test_utils_python.py @@ -22,6 +22,9 @@ class UtilsPythonTestCase(unittest.TestCase): # converting a strange object should raise TypeError self.assertRaises(TypeError, str_to_unicode, 423) + # check errors argument works + self.assertEqual(str_to_unicode('a\xedb', 'utf-8', errors='replace'), u'a\ufffdb') + def test_unicode_to_str(self): # converting a unicode object to an utf-8 encoded string self.assertEqual(unicode_to_str(u'\xa3 49'), '\xc2\xa3 49') @@ -35,6 +38,9 @@ class UtilsPythonTestCase(unittest.TestCase): # converting a strange object should raise TypeError self.assertRaises(TypeError, unicode_to_str, unittest) + # check errors argument works + (str_to_unicode('a\xedb', 'latin-', errors='replace'), u'a?b') + def test_memoizemethod_noargs(self): class A(object): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 486aece73..9a0610c32 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -63,7 +63,7 @@ def unique(list_, key=lambda x: x): return result -def str_to_unicode(text, encoding=None): +def str_to_unicode(text, encoding=None, errors='strict'): """Return the unicode representation of text in the given encoding. Unlike .encode(encoding) this function can be applied directly to a unicode object without the risk of double-decoding problems (which can happen if @@ -73,13 +73,13 @@ def str_to_unicode(text, encoding=None): if encoding is None: encoding = 'utf-8' if isinstance(text, str): - return text.decode(encoding) + return text.decode(encoding, errors) elif isinstance(text, unicode): return text else: raise TypeError('str_to_unicode must receive a str or unicode object, got %s' % type(text).__name__) -def unicode_to_str(text, encoding=None): +def unicode_to_str(text, encoding=None, errors='strict'): """Return the str representation of text in the given encoding. Unlike .encode(encoding) this function can be applied directly to a str object without the risk of double-decoding problems (which can happen if @@ -89,7 +89,7 @@ def unicode_to_str(text, encoding=None): if encoding is None: encoding = 'utf-8' if isinstance(text, unicode): - return text.encode(encoding) + return text.encode(encoding, errors) elif isinstance(text, str): return text else: