From 92780b70da60bbb328b534c1b43e45314713e39a Mon Sep 17 00:00:00 2001 From: Ruben Vereecken Date: Wed, 19 Mar 2014 21:48:44 +0100 Subject: [PATCH 1/2] Added content-type check as per issue #193 --- scrapy/contrib/downloadermiddleware/httpcompression.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py index b03a29d67..719507396 100644 --- a/scrapy/contrib/downloadermiddleware/httpcompression.py +++ b/scrapy/contrib/downloadermiddleware/httpcompression.py @@ -1,6 +1,6 @@ import zlib -from scrapy.utils.gz import gunzip +from scrapy.utils.gz import gunzip, is_gzipped from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.exceptions import NotConfigured @@ -22,7 +22,7 @@ class HttpCompressionMiddleware(object): def process_response(self, request, response, spider): if isinstance(response, Response): content_encoding = response.headers.getlist('Content-Encoding') - if content_encoding: + if content_encoding and not is_gzipped(response): encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) respcls = responsetypes.from_args(headers=response.headers, \ From 54434095fc03d5ad12694779f9e476ded28764ed Mon Sep 17 00:00:00 2001 From: Ruben Vereecken Date: Fri, 21 Mar 2014 14:41:07 +0100 Subject: [PATCH 2/2] Redefined test for #193 --- .../tests/test_downloadermiddleware_httpcompression.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py index 770fa7c33..5fbc2c619 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -135,3 +135,12 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding('gb2312')) + def test_process_response_gzipped_contenttype(self): + response = self._getresponse('gzip') + response.headers['Content-Type'] = 'application/gzip' + request = response.request + + newresponse = self.mw.process_response(request, response, self.spider) + self.assertIs(newresponse, response) + self.assertEqual(response.headers['Content-Encoding'], 'gzip') + self.assertEqual(response.headers['Content-Type'], 'application/gzip')