diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpcompression.py index 3ce10f914..a2de8757d 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpcompression.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpcompression.py @@ -14,13 +14,14 @@ class HttpCompressionMiddleware(object): def process_response(self, request, response, spider): if isinstance(response, Response): - content_encoding = response.headers.get('Content-Encoding') + content_encoding = response.headers.getlist('Content-Encoding') if content_encoding: - encoding = content_encoding[0].lower() - raw_body = response.body - decoded_body = self._decode(raw_body, encoding) + encoding = content_encoding.pop() + decoded_body = self._decode(response.body, encoding.lower()) response = response.replace(body=decoded_body) - response.headers['Content-Encoding'] = content_encoding[1:] + if not content_encoding: + del response.headers['Content-Encoding'] + return response def _decode(self, body, encoding): diff --git a/scrapy/trunk/scrapy/tests/sample_data/compressed/html-gzip.bin b/scrapy/trunk/scrapy/tests/sample_data/compressed/html-gzip.bin new file mode 100644 index 000000000..5e4f489b4 Binary files /dev/null and b/scrapy/trunk/scrapy/tests/sample_data/compressed/html-gzip.bin differ diff --git a/scrapy/trunk/scrapy/tests/sample_data/compressed/html-rawdeflate.bin b/scrapy/trunk/scrapy/tests/sample_data/compressed/html-rawdeflate.bin new file mode 100644 index 000000000..14e5cda2a Binary files /dev/null and b/scrapy/trunk/scrapy/tests/sample_data/compressed/html-rawdeflate.bin differ diff --git a/scrapy/trunk/scrapy/tests/sample_data/compressed/html-zlibdeflate.bin b/scrapy/trunk/scrapy/tests/sample_data/compressed/html-zlibdeflate.bin new file mode 100644 index 000000000..b4554a979 Binary files /dev/null and b/scrapy/trunk/scrapy/tests/sample_data/compressed/html-zlibdeflate.bin differ diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_httpcompression.py new file mode 100644 index 000000000..3d1c9f9c0 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -0,0 +1,99 @@ +from __future__ import with_statement + +from unittest import TestCase +from os.path import join, abspath, dirname + +from scrapy.spider import spiders +from scrapy.http import Response, Request +from scrapy.contrib.downloadermiddleware.httpcompression import HttpCompressionMiddleware + + +SAMPLEDIR = join(abspath(dirname(__file__)), 'sample_data/compressed') + +FORMAT = { + 'gzip': ('html-gzip.bin', 'gzip'), + 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), + 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), + } + +class HttpCompressionTest(TestCase): + + def setUp(self): + spiders.spider_modules = ['scrapy.tests.test_spiders'] + spiders.reload() + self.spider = spiders.fromdomain('scrapytest.org') + self.mw = HttpCompressionMiddleware() + + def _getresponse(self, coding): + if coding not in FORMAT: + raise ValueError() + + samplefile, contentencoding = FORMAT[coding] + + with open(join(SAMPLEDIR, samplefile), 'rb') as sample: + body = sample.read() + + headers = { + 'Server': 'Yaws/1.49 Yet Another Web Server', + 'Date': 'Sun, 08 Mar 2009 00:41:03 GMT', + 'Content-Length': len(body), + 'Content-Type': 'text/html', + 'Content-Encoding': contentencoding, + } + + response = Response('http://scrapytest.org/', body=body, headers=headers) + response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip,deflate'}) + return response + + def test_process_request(self): + request = Request('http://scrapytest.org') + assert 'Accept-Encoding' not in request.headers + self.mw.process_request(request, self.spider) + self.assertEqual(request.headers.get('Accept-Encoding'), 'gzip,deflate') + + def test_process_response_gzip(self): + response = self._getresponse('gzip') + request = response.request + + self.assertEqual(response.headers['Content-Encoding'], 'gzip') + newresponse = self.mw.process_response(request, response, self.spider) + assert newresponse is not response + assert newresponse.body.startswith('