Merge pull request #2050 from Tethik/is_gzipped_fix

[MRG+1] Is_gzipped for application/x-gzip;charset=utf-8

scrapy/utils/gz.py

@@ -7,7 +7,7 @@ except ImportError:
 from gzip import GzipFile
 
 import six
-
+import re
 
 # - Python>=3.5 GzipFile's read() has issues returning leftover
 #   uncompressed data when input is corrupted
@@ -50,8 +50,9 @@ def gunzip(data):
                 raise
     return output
 
+_is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I)
 
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')
-    return ctype in (b'application/x-gzip', b'application/gzip')
+    return _is_gzipped_re.search(ctype) is not None

tests/test_utils_gz.py

@@ -1,7 +1,8 @@
 import unittest
 from os.path import join
 
-from scrapy.utils.gz import gunzip
+from scrapy.utils.gz import gunzip, is_gzipped
+from scrapy.http import Response, Headers
 from tests import tests_datadir
 
 SAMPLEDIR = join(tests_datadir, 'compressed')
@@ -27,3 +28,41 @@ class GunzipTest(unittest.TestCase):
         with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f:
             text = gunzip(f.read())
             assert text.endswith(b'</html>')
+
+    def test_is_x_gzipped_right(self):
+        hdrs = Headers({"Content-Type": "application/x-gzip"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(is_gzipped(r1))
+
+    def test_is_gzipped_right(self):
+        hdrs = Headers({"Content-Type": "application/gzip"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(is_gzipped(r1))
+
+    def test_is_gzipped_not_quite(self):
+        hdrs = Headers({"Content-Type": "application/gzippppp"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertFalse(is_gzipped(r1))
+
+    def test_is_gzipped_case_insensitive(self):
+        hdrs = Headers({"Content-Type": "Application/X-Gzip"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(is_gzipped(r1))
+
+        hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(is_gzipped(r1))
+
+    def test_is_gzipped_empty(self):
+        r1 = Response("http://www.example.com")
+        self.assertFalse(is_gzipped(r1))
+
+    def test_is_gzipped_wrong(self):
+        hdrs = Headers({"Content-Type": "application/javascript"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertFalse(is_gzipped(r1))
+
+    def test_is_gzipped_with_charset(self):
+        hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"})
+        r1 = Response("http://www.example.com", headers=hdrs)
+        self.assertTrue(is_gzipped(r1))