Merge pull request #660 from rubenvereecken/issue193

Added content-type check as per issue #193

scrapy/contrib/downloadermiddleware/httpcompression.py

@@ -1,6 +1,6 @@
 import zlib
 
-from scrapy.utils.gz import gunzip
+from scrapy.utils.gz import gunzip, is_gzipped
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.exceptions import NotConfigured
@@ -22,7 +22,7 @@ class HttpCompressionMiddleware(object):
     def process_response(self, request, response, spider):
         if isinstance(response, Response):
             content_encoding = response.headers.getlist('Content-Encoding')
-            if content_encoding:
+            if content_encoding and not is_gzipped(response):
                 encoding = content_encoding.pop()
                 decoded_body = self._decode(response.body, encoding.lower())
                 respcls = responsetypes.from_args(headers=response.headers, \

scrapy/tests/test_downloadermiddleware_httpcompression.py

@@ -135,3 +135,12 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
 
+    def test_process_response_gzipped_contenttype(self):
+        response = self._getresponse('gzip')
+        response.headers['Content-Type'] = 'application/gzip'
+        request = response.request
+
+        newresponse = self.mw.process_response(request, response, self.spider)
+        self.assertIs(newresponse, response)
+        self.assertEqual(response.headers['Content-Encoding'], 'gzip')
+        self.assertEqual(response.headers['Content-Type'], 'application/gzip')