diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index cfb652143..afc7ed128 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -50,9 +50,12 @@ def gunzip(data): raise return output -_is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I) +_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search +_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') - return _is_gzipped_re.search(ctype) is not None + cenc = response.headers.get('Content-Encoding', b'').lower() + return (_is_gzipped(ctype) or + (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip'))) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 24955a515..b2426946d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -145,6 +145,26 @@ class HttpCompressionTest(TestCase): self.assertEqual(response.headers['Content-Encoding'], b'gzip') self.assertEqual(response.headers['Content-Type'], b'application/gzip') + def test_process_response_gzip_app_octetstream_contenttype(self): + response = self._getresponse('gzip') + response.headers['Content-Type'] = 'application/octet-stream' + request = response.request + + newresponse = self.mw.process_response(request, response, self.spider) + self.assertIs(newresponse, response) + self.assertEqual(response.headers['Content-Encoding'], b'gzip') + self.assertEqual(response.headers['Content-Type'], b'application/octet-stream') + + def test_process_response_gzip_binary_octetstream_contenttype(self): + response = self._getresponse('x-gzip') + response.headers['Content-Type'] = 'binary/octet-stream' + request = response.request + + newresponse = self.mw.process_response(request, response, self.spider) + self.assertIs(newresponse, response) + self.assertEqual(response.headers['Content-Encoding'], b'gzip') + self.assertEqual(response.headers['Content-Type'], b'binary/octet-stream') + def test_process_response_head_request_no_decode_required(self): response = self._getresponse('gzip') response.headers['Content-Type'] = 'application/gzip'