From 10a2c46e12e54e666de743b9ceca09c18370ccec Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 20 Jun 2016 13:39:37 +0200
Subject: [PATCH] [HttpCompressionMiddleware] Do not decompress
 binary/octet-stream responses

---
 scrapy/utils/gz.py                            |  9 +++++++--
 ...st_downloadermiddleware_httpcompression.py | 20 +++++++++++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index d035f9fdf..b2f737193 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -53,5 +53,10 @@ def gunzip(data):
 
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
-    ctype = response.headers.get('Content-Type', b'')
-    return ctype in (b'application/x-gzip', b'application/gzip')
+    ctype = response.headers.get('Content-Type', b'').lower()
+    cenc = response.headers.get('Content-Encoding', b'').lower()
+    return (
+        ctype in (b'application/x-gzip', b'application/gzip') or
+        (ctype in (b'application/octet-stream', b'binary/octet-stream') and
+         cenc in (b'gzip', b'x-gzip'))
+    )
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 24955a515..b2426946d 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -145,6 +145,26 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.headers['Content-Encoding'], b'gzip')
         self.assertEqual(response.headers['Content-Type'], b'application/gzip')
 
+    def test_process_response_gzip_app_octetstream_contenttype(self):
+        response = self._getresponse('gzip')
+        response.headers['Content-Type'] = 'application/octet-stream'
+        request = response.request
+
+        newresponse = self.mw.process_response(request, response, self.spider)
+        self.assertIs(newresponse, response)
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers['Content-Type'], b'application/octet-stream')
+
+    def test_process_response_gzip_binary_octetstream_contenttype(self):
+        response = self._getresponse('x-gzip')
+        response.headers['Content-Type'] = 'binary/octet-stream'
+        request = response.request
+
+        newresponse = self.mw.process_response(request, response, self.spider)
+        self.assertIs(newresponse, response)
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers['Content-Type'], b'binary/octet-stream')
+
     def test_process_response_head_request_no_decode_required(self):
         response = self._getresponse('gzip')
         response.headers['Content-Type'] = 'application/gzip'