From bda448f17db418d1829621e1c53b1cbe69771bc9 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Mon, 20 Jun 2016 13:39:37 +0200
Subject: [PATCH] [HttpCompressionMiddleware] Do not decompress
 binary/octet-stream responses

---
 scrapy/utils/gz.py                            |  7 +++++--
 ...st_downloadermiddleware_httpcompression.py | 20 +++++++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index cfb652143..afc7ed128 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -50,9 +50,12 @@ def gunzip(data):
                 raise
     return output
 
-_is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I)
+_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
+_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
 
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')
-    return _is_gzipped_re.search(ctype) is not None
+    cenc = response.headers.get('Content-Encoding', b'').lower()
+    return (_is_gzipped(ctype) or
+            (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 24955a515..b2426946d 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -145,6 +145,26 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.headers['Content-Encoding'], b'gzip')
         self.assertEqual(response.headers['Content-Type'], b'application/gzip')
 
+    def test_process_response_gzip_app_octetstream_contenttype(self):
+        response = self._getresponse('gzip')
+        response.headers['Content-Type'] = 'application/octet-stream'
+        request = response.request
+
+        newresponse = self.mw.process_response(request, response, self.spider)
+        self.assertIs(newresponse, response)
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers['Content-Type'], b'application/octet-stream')
+
+    def test_process_response_gzip_binary_octetstream_contenttype(self):
+        response = self._getresponse('x-gzip')
+        response.headers['Content-Type'] = 'binary/octet-stream'
+        request = response.request
+
+        newresponse = self.mw.process_response(request, response, self.spider)
+        self.assertIs(newresponse, response)
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers['Content-Type'], b'binary/octet-stream')
+
     def test_process_response_head_request_no_decode_required(self):
         response = self._getresponse('gzip')
         response.headers['Content-Type'] = 'application/gzip'