Merge pull request #2043 from scrapy/backport-1.1-pr2008

[backport][1.1] HttpCompressionMiddleware: don't decode HEAD responses (PR #2008)

scrapy/downloadermiddlewares/httpcompression.py

@@ -20,6 +20,9 @@ class HttpCompressionMiddleware(object):
         request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
 
     def process_response(self, request, response, spider):
+
+        if request.method == 'HEAD':
+            return response
         if isinstance(response, Response):
             content_encoding = response.headers.getlist('Content-Encoding')
             if content_encoding and not is_gzipped(response):

tests/test_downloadermiddleware_httpcompression.py

@@ -144,3 +144,13 @@ class HttpCompressionTest(TestCase):
         self.assertIs(newresponse, response)
         self.assertEqual(response.headers['Content-Encoding'], b'gzip')
         self.assertEqual(response.headers['Content-Type'], b'application/gzip')
+
+    def test_process_response_head_request_no_decode_required(self):
+        response = self._getresponse('gzip')
+        response.headers['Content-Type'] = 'application/gzip'
+        request = response.request
+        request.method = 'HEAD'
+        response = response.replace(body = None)
+        newresponse = self.mw.process_response(request, response, self.spider)
+        self.assertIs(newresponse, response)
+        self.assertEquals(response.body, b'')