diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py
index a2de8757d..031001f15 100644
--- a/scrapy/contrib/downloadermiddleware/httpcompression.py
+++ b/scrapy/contrib/downloadermiddleware/httpcompression.py
@@ -3,6 +3,7 @@ from gzip import GzipFile
from cStringIO import StringIO
from scrapy.http import Response
+from scrapy.core.downloader.responsetypes import responsetypes
class HttpCompressionMiddleware(object):
@@ -18,7 +19,9 @@ class HttpCompressionMiddleware(object):
if content_encoding:
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
- response = response.replace(body=decoded_body)
+ respcls = responsetypes.from_args(headers=response.headers, \
+ url=response.url)
+ response = response.replace(cls=respcls, body=decoded_body)
if not content_encoding:
del response.headers['Content-Encoding']
diff --git a/scrapy/core/downloader/responsetypes/__init__.py b/scrapy/core/downloader/responsetypes/__init__.py
index 297ec30ac..51631b282 100644
--- a/scrapy/core/downloader/responsetypes/__init__.py
+++ b/scrapy/core/downloader/responsetypes/__init__.py
@@ -46,9 +46,11 @@ class ResponseTypes(object):
basetype = "%s/*" % mimetype.split('/')[0]
return self.classes.get(basetype, Response)
- def from_content_type(self, content_type):
+ def from_content_type(self, content_type, content_encoding=None):
"""Return the most appropiate Response class from an HTTP Content-Type
header """
+ if content_encoding:
+ return Response
mimetype = content_type.split(';')[0].strip().lower()
return self.from_mimetype(mimetype)
@@ -65,7 +67,8 @@ class ResponseTypes(object):
headers"""
cls = Response
if 'Content-Type' in headers:
- cls = self.from_content_type(headers['Content-type'])
+ cls = self.from_content_type(headers['Content-type'], \
+ headers.get('Content-Encoding'))
if cls is Response and 'Content-Disposition' in headers:
cls = self.from_content_disposition(headers['Content-Disposition'])
return cls
diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py
index c8fb38d22..40988ecd7 100644
--- a/scrapy/tests/test_downloadermiddleware_httpcompression.py
+++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py
@@ -2,11 +2,14 @@ from __future__ import with_statement
from unittest import TestCase
from os.path import join, abspath, dirname
+from cStringIO import StringIO
+from gzip import GzipFile
from scrapy.spider import BaseSpider
-from scrapy.http import Response, Request
+from scrapy.http import Response, Request, HtmlResponse
from scrapy.contrib.downloadermiddleware.httpcompression import HttpCompressionMiddleware
from scrapy.tests import tests_datadir
+from scrapy.utils.encoding import resolve_encoding
SAMPLEDIR = join(tests_datadir, 'compressed')
@@ -96,3 +99,22 @@ class HttpCompressionTest(TestCase):
newresponse = self.mw.process_response(request, response, self.spider)
assert newresponse is not response
self.assertEqual(newresponse.headers.getlist('Content-Encoding'), ['uuencode'])
+
+ def test_process_response_encoding_inside_body(self):
+ headers = {
+ 'Content-Type': 'text/html',
+ 'Content-Encoding': 'gzip',
+ }
+ f = StringIO()
+ plainbody = """Some page"""
+ zf = GzipFile(fileobj=f, mode='wb')
+ zf.write(plainbody)
+ zf.close()
+ response = Response("http;//www.example.com/", headers=headers, body=f.getvalue())
+ request = Request("http://www.example.com/")
+
+ newresponse = self.mw.process_response(request, response, self.spider)
+ assert isinstance(newresponse, HtmlResponse)
+ self.assertEqual(newresponse.body, plainbody)
+ self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
+
diff --git a/scrapy/tests/test_responsetypes.py b/scrapy/tests/test_responsetypes.py
index 14f3cf387..8e4e1643a 100644
--- a/scrapy/tests/test_responsetypes.py
+++ b/scrapy/tests/test_responsetypes.py
@@ -54,6 +54,7 @@ class ResponseTypesTest(unittest.TestCase):
mappings = [
({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse),
({'Content-Type': ['application/octet-stream'], 'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse),
+ ({'Content-Type': ['text/html; charset=utf-8'], 'Content-Encoding': ['gzip']}, Response),
]
for source, cls in mappings:
source = Headers(source)