From a9c73601ca84ca999cf2a60f8a3733da90c994f6 Mon Sep 17 00:00:00 2001 From: omer Date: Mon, 23 May 2016 19:52:38 +0300 Subject: [PATCH 1/3] Do not decode of head response --- tests/test_downloadermiddleware_httpcompression.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 2e6e47fef..9ef66ba1c 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -144,3 +144,13 @@ class HttpCompressionTest(TestCase): self.assertIs(newresponse, response) self.assertEqual(response.headers['Content-Encoding'], b'gzip') self.assertEqual(response.headers['Content-Type'], b'application/gzip') + + def test_process_response_head_request_no_decode_required(self): + response = self._getresponse('gzip') + response.headers['Content-Type'] = 'application/gzip' + request = response.request + request.method = 'HEAD' + response = response.replace(body = None) + newresponse = self.mw.process_response(request, response, self.spider) + self.assertIs(newresponse, response) + self.assertEquals(response.body, '') From 9e3da69867c8466deffafa66283bca113e2187ee Mon Sep 17 00:00:00 2001 From: omer Date: Tue, 24 May 2016 14:24:29 +0300 Subject: [PATCH 2/3] Do not decode of head response (2) --- scrapy/downloadermiddlewares/httpcompression.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 7ab304c17..bcf20f10c 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -20,6 +20,9 @@ class HttpCompressionMiddleware(object): request.headers.setdefault('Accept-Encoding', 'gzip,deflate') def process_response(self, request, response, spider): + + if request.method == 'HEAD': + return response if isinstance(response, Response): content_encoding = response.headers.getlist('Content-Encoding') if content_encoding and not is_gzipped(response): From 06aee3f6b390de5c08151a6bdb7c8c611f7122b1 Mon Sep 17 00:00:00 2001 From: omer Date: Wed, 25 May 2016 06:56:06 +0300 Subject: [PATCH 3/3] fix issue with '' in python 3 --- tests/test_downloadermiddleware_httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 9ef66ba1c..24955a515 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -153,4 +153,4 @@ class HttpCompressionTest(TestCase): response = response.replace(body = None) newresponse = self.mw.process_response(request, response, self.spider) self.assertIs(newresponse, response) - self.assertEquals(response.body, '') + self.assertEquals(response.body, b'')