added some encoding aliases not provided in Python by default - fixes #130

scrapy/__init__.py

@@ -14,6 +14,11 @@ if sys.version_info < (2,5):
 # monkey patches to fix external library issues
 from scrapy.xlib import twisted_250_monkeypatches
 
+# add some common encoding aliases not included by default in Python
+from scrapy.utils.encoding import add_encoding_alias
+add_encoding_alias('gb2312', 'zh-cn')
+add_encoding_alias('cp1251', 'win-1251')
+
 # optional_features is a set containing Scrapy optional features
 optional_features = set()
 

scrapy/tests/test_encoding_aliases.py

@@ -0,0 +1,21 @@
+import unittest
+
+import scrapy # adds encoding aliases (if not added before)
+
+class EncodingAliasesTestCase(unittest.TestCase):
+
+    def test_encoding_aliases(self):
+        """Test common encdoing aliases not included in Python"""
+
+        uni = u'\u041c\u041e\u0421K\u0412\u0410'
+        str = uni.encode('windows-1251')
+        self.assertEqual(uni.encode('windows-1251'), uni.encode('win-1251'))
+        self.assertEqual(str.decode('windows-1251'), str.decode('win-1251'))
+
+        text = u'\u8f6f\u4ef6\u540d\u79f0'
+        str = uni.encode('gb2312')
+        self.assertEqual(uni.encode('gb2312'), uni.encode('zh-cn'))
+        self.assertEqual(str.decode('gb2312'), str.decode('zh-cn'))
+
+if __name__ == "__main__":
+    unittest.main()

scrapy/utils/encoding.py

@@ -0,0 +1,11 @@
+import codecs
+
+def add_encoding_alias(encoding, alias, overwrite=False):
+    try:
+        codecs.lookup(alias)
+        alias_exists = True
+    except LookupError:
+        alias_exists = False
+    if overwrite or not alias_exists:
+        codec = codecs.lookup(encoding)
+        codecs.register(lambda x: codec if x == alias else None)