From 10a2c46e12e54e666de743b9ceca09c18370ccec Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 20 Jun 2016 13:39:37 +0200 Subject: [PATCH 1/2] [HttpCompressionMiddleware] Do not decompress binary/octet-stream responses --- scrapy/utils/gz.py | 9 +++++++-- ...st_downloadermiddleware_httpcompression.py | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index d035f9fdf..b2f737193 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -53,5 +53,10 @@ def gunzip(data): def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" - ctype = response.headers.get('Content-Type', b'') - return ctype in (b'application/x-gzip', b'application/gzip') + ctype = response.headers.get('Content-Type', b'').lower() + cenc = response.headers.get('Content-Encoding', b'').lower() + return ( + ctype in (b'application/x-gzip', b'application/gzip') or + (ctype in (b'application/octet-stream', b'binary/octet-stream') and + cenc in (b'gzip', b'x-gzip')) + ) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 24955a515..b2426946d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -145,6 +145,26 @@ class HttpCompressionTest(TestCase): self.assertEqual(response.headers['Content-Encoding'], b'gzip') self.assertEqual(response.headers['Content-Type'], b'application/gzip') + def test_process_response_gzip_app_octetstream_contenttype(self): + response = self._getresponse('gzip') + response.headers['Content-Type'] = 'application/octet-stream' + request = response.request + + newresponse = self.mw.process_response(request, response, self.spider) + self.assertIs(newresponse, response) + self.assertEqual(response.headers['Content-Encoding'], b'gzip') + self.assertEqual(response.headers['Content-Type'], b'application/octet-stream') + + def test_process_response_gzip_binary_octetstream_contenttype(self): + response = self._getresponse('x-gzip') + response.headers['Content-Type'] = 'binary/octet-stream' + request = response.request + + newresponse = self.mw.process_response(request, response, self.spider) + self.assertIs(newresponse, response) + self.assertEqual(response.headers['Content-Encoding'], b'gzip') + self.assertEqual(response.headers['Content-Type'], b'binary/octet-stream') + def test_process_response_head_request_no_decode_required(self): response = self._getresponse('gzip') response.headers['Content-Type'] = 'application/gzip' From 962eb11c73bdb56b81e96eddccde189a22f43963 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 11 Jul 2016 11:30:05 +0200 Subject: [PATCH 2/2] Simplify regex conditions --- scrapy/utils/gz.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 9fe88d108..afc7ed128 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -50,13 +50,12 @@ def gunzip(data): raise return output -_is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I) -_is_octetstream_re = re.compile(br'^(application|binary)/octet-stream\b', re.I) +_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search +_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') cenc = response.headers.get('Content-Encoding', b'').lower() - return (_is_gzipped_re.search(ctype) is not None or - (_is_octetstream_re.search(ctype) is not None and - cenc in (b'gzip', b'x-gzip'))) + return (_is_gzipped(ctype) or + (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))