From 27cbcbcdafd7eba268ad0e35a58e6e12a8db1da9 Mon Sep 17 00:00:00 2001 From: Joakim Uddholm Date: Tue, 14 Jun 2016 14:39:16 +0200 Subject: [PATCH] is_gzipped: Changed to regex to check the content-type header. Also suggested by @redapple. --- scrapy/utils/gz.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index f174950a4..f2a9555b1 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -7,7 +7,7 @@ except ImportError: from gzip import GzipFile import six - +import re # - Python>=3.5 GzipFile's read() has issues returning leftover # uncompressed data when input is corrupted @@ -50,8 +50,9 @@ def gunzip(data): raise return output +_is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I) def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') - return b'application/x-gzip' in ctype or b'application/gzip' in ctype + return not _is_gzipped_re.search(ctype) is None