From 0b07742adb77e34982ebd9a5c9f32b8e8aad737d Mon Sep 17 00:00:00 2001
From: Daniel Grana <dangra@gmail.com>
Date: Mon, 5 Apr 2010 15:07:43 -0300
Subject: [PATCH] gb2312 and gbk encodings was superseded by gb18030

---
 scrapy/conf/default_settings.py    | 27 ++++++++++++++++++++++-----
 scrapy/tests/test_http_response.py |  4 ++++
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py
index 5e7bea3cd..b80e89385 100644
--- a/scrapy/conf/default_settings.py
+++ b/scrapy/conf/default_settings.py
@@ -74,18 +74,35 @@ DUPEFILTER_CLASS = 'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter'
 ENCODING_ALIASES = {}
 
 ENCODING_ALIASES_BASE = {
-    'zh-cn': 'gb2312',
-    'win-1251': 'cp1251',
-    'macintosh' : 'mac-roman',
-    'x-sjis': 'shift-jis',
+    # gb2312 is superseded by gb18030
+    'gb2312': 'gb18030',
+    'chinese': 'gb18030',
+    'csiso58gb231280': 'gb18030',
+    'euc- cn': 'gb18030',
+    'euccn': 'gb18030',
+    'eucgb2312-cn': 'gb18030',
+    'gb2312-1980': 'gb18030',
+    'gb2312-80': 'gb18030',
+    'iso- ir-58': 'gb18030',
+    # gbk is superseded by gb18030
+    'gbk': 'gb18030',
+    '936': 'gb18030',
+    'cp936': 'gb18030',
+    'ms936': 'gb18030',
+    # latin_1 is a subset of cp1252
+    'latin_1': 'cp1252',
     'iso-8859-1': 'cp1252',
     'iso8859-1': 'cp1252',
     '8859': 'cp1252',
     'cp819': 'cp1252',
     'latin': 'cp1252',
     'latin1': 'cp1252',
-    'latin_1': 'cp1252',
     'l1': 'cp1252',
+    # others
+    'zh-cn': 'gb18030',
+    'win-1251': 'cp1251',
+    'macintosh' : 'mac_roman',
+    'x-sjis': 'shift_jis',
 }
 
 EXTENSIONS = {}
diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py
index d17d5d272..87138219b 100644
--- a/scrapy/tests/test_http_response.py
+++ b/scrapy/tests/test_http_response.py
@@ -179,6 +179,8 @@ class TextResponseTest(BaseResponseTest):
         r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3")
         r4 = self.response_class("http://www.example.com", body="\xa2\xa3")
         r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3")
+        r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body="\xa8D")
+        r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body="\xa8D")
 
         self.assertEqual(r1._headers_encoding(), "utf-8")
         self.assertEqual(r2._headers_encoding(), None)
@@ -193,6 +195,8 @@ class TextResponseTest(BaseResponseTest):
         self._assert_response_values(r1, 'utf-8', u"\xa3")
         self._assert_response_values(r2, 'utf-8', u"\xa3")
         self._assert_response_values(r3, 'iso-8859-1', u"\xa3")
+        self._assert_response_values(r6, 'gb18030', u"\u2015")
+        self._assert_response_values(r7, 'gb18030', u"\u2015")
 
         # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
         self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3")