RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Force recalculating the encoding on HTTPCompression middleware until we are sure the responsetypes guessing is reliable. Refs #239, #240

This commit is contained in:
Pablo Hoffman 2010-09-15 13:57:17 -03:00
parent 28f6bdf7bb
commit 5fe9a49e51
2 changed files with 25 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrapy/contrib/downloadermiddleware/httpcompression.py
View File

@ -2,7 +2,7 @@ import zlib
from gzip import GzipFile
from cStringIO import StringIO
from scrapy.http import Response
from scrapy.http import Response, TextResponse
from scrapy.core.downloader.responsetypes import responsetypes
@ -21,7 +21,12 @@ class HttpCompressionMiddleware(object):
decoded_body = self._decode(response.body, encoding.lower())
respcls = responsetypes.from_args(headers=response.headers, \
url=response.url)
response = response.replace(cls=respcls, body=decoded_body)
kwargs = dict(cls=respcls, body=decoded_body)
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
# responsetypes guessing is reliable
kwargs['encoding'] = None
response = response.replace(**kwargs)
if not content_encoding:
del response.headers['Content-Encoding']

18
scrapy/tests/test_downloadermiddleware_httpcompression.py
View File

@ -118,3 +118,21 @@ class HttpCompressionTest(TestCase):
self.assertEqual(newresponse.body, plainbody)
self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
def test_process_response_force_recalculate_encoding(self):
headers = {
'Content-Type': 'text/html',
'Content-Encoding': 'gzip',
}
f = StringIO()
plainbody = """<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
zf = GzipFile(fileobj=f, mode='wb')
zf.write(plainbody)
zf.close()
response = HtmlResponse("http;//www.example.com/page.html", headers=headers, body=f.getvalue())
request = Request("http://www.example.com/")
newresponse = self.mw.process_response(request, response, self.spider)
assert isinstance(newresponse, HtmlResponse)
self.assertEqual(newresponse.body, plainbody)
self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))