diff --git a/scrapy/__init__.py b/scrapy/__init__.py index bf6d46e17..0051aa75e 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -14,6 +14,11 @@ if sys.version_info < (2,5): # monkey patches to fix external library issues from scrapy.xlib import twisted_250_monkeypatches +# add some common encoding aliases not included by default in Python +from scrapy.utils.encoding import add_encoding_alias +add_encoding_alias('gb2312', 'zh-cn') +add_encoding_alias('cp1251', 'win-1251') + # optional_features is a set containing Scrapy optional features optional_features = set() diff --git a/scrapy/tests/test_encoding_aliases.py b/scrapy/tests/test_encoding_aliases.py new file mode 100644 index 000000000..cc3a0089f --- /dev/null +++ b/scrapy/tests/test_encoding_aliases.py @@ -0,0 +1,21 @@ +import unittest + +import scrapy # adds encoding aliases (if not added before) + +class EncodingAliasesTestCase(unittest.TestCase): + + def test_encoding_aliases(self): + """Test common encdoing aliases not included in Python""" + + uni = u'\u041c\u041e\u0421K\u0412\u0410' + str = uni.encode('windows-1251') + self.assertEqual(uni.encode('windows-1251'), uni.encode('win-1251')) + self.assertEqual(str.decode('windows-1251'), str.decode('win-1251')) + + text = u'\u8f6f\u4ef6\u540d\u79f0' + str = uni.encode('gb2312') + self.assertEqual(uni.encode('gb2312'), uni.encode('zh-cn')) + self.assertEqual(str.decode('gb2312'), str.decode('zh-cn')) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/utils/encoding.py b/scrapy/utils/encoding.py new file mode 100644 index 000000000..9eb06d942 --- /dev/null +++ b/scrapy/utils/encoding.py @@ -0,0 +1,11 @@ +import codecs + +def add_encoding_alias(encoding, alias, overwrite=False): + try: + codecs.lookup(alias) + alias_exists = True + except LookupError: + alias_exists = False + if overwrite or not alias_exists: + codec = codecs.lookup(encoding) + codecs.register(lambda x: codec if x == alias else None)