diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index eb00d8923..203dee42d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,6 @@ import zlib -from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.exceptions import NotConfigured @@ -34,7 +34,7 @@ class HttpCompressionMiddleware(object): return response if isinstance(response, Response): content_encoding = response.headers.getlist('Content-Encoding') - if content_encoding and not is_gzipped(response): + if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) respcls = responsetypes.from_args(headers=response.headers, \ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 9e45637c3..e54001d88 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -5,7 +5,8 @@ import six from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots -from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.utils.gz import gunzip, gzip_magic_number + logger = logging.getLogger(__name__) @@ -59,12 +60,19 @@ class SitemapSpider(Spider): """ if isinstance(response, XmlResponse): return response.body - elif is_gzipped(response): + elif gzip_magic_number(response): return gunzip(response.body) - elif response.url.endswith('.xml'): + # actual gzipped sitemap files are decompressed above ; + # if we are here (response body is not gzipped) + # and have a response for .xml.gz, + # it usually means that it was already gunzipped + # by HttpCompression middleware, + # the HTTP response being sent with "Content-Encoding: gzip" + # without actually being a .xml.gz file in the first place, + # merely XML gzip-compressed on the fly, + # in other word, here, we have plain XML + elif response.url.endswith('.xml') or response.url.endswith('.xml.gz'): return response.body - elif response.url.endswith('.xml.gz'): - return gunzip(response.body) def regex(x): diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 73c2eb73b..16c9ce539 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -59,3 +59,7 @@ def is_gzipped(response): cenc = response.headers.get('Content-Encoding', b'').lower() return (_is_gzipped(ctype) or (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip'))) + + +def gzip_magic_number(response): + return response.body[:3] == b'\x1f\x8b\x08' diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5403e8f52..0678fcb14 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -8,6 +8,7 @@ from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ ACCEPTED_ENCODINGS from scrapy.responsetypes import responsetypes +from scrapy.utils.gz import gunzip from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -173,9 +174,9 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) - self.assertIs(newresponse, response) - self.assertEqual(response.headers['Content-Encoding'], b'gzip') - self.assertEqual(response.headers['Content-Type'], b'application/gzip') + self.assertIsNot(newresponse, response) + self.assertTrue(newresponse.body.startswith(b' + + + http://www.example.com/ + 2009-08-16 + daily + 1 + + + http://www.example.com/Special-Offers.html + 2009-08-16 + weekly + 0.8 + +""" + gz_file = GzipFile(fileobj=f, mode='wb') + gz_file.write(plainbody) + gz_file.close() + + # build a gzipped response body containing this gzipped file + r = BytesIO() + gz_resp = GzipFile(fileobj=r, mode='wb') + gz_resp.write(f.getvalue()) + gz_resp.close() + + response = Response("http;//www.example.com/", headers=headers, body=r.getvalue()) + request = Request("http://www.example.com/") + + newresponse = self.mw.process_response(request, response, self.spider) + self.assertEqual(gunzip(newresponse.body), plainbody) def test_process_response_head_request_no_decode_required(self): response = self._getresponse('gzip') diff --git a/tests/test_spider.py b/tests/test_spider.py index 371b8c1ac..e55f0fa6d 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -328,6 +328,10 @@ class SitemapSpiderTest(SpiderTest): r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) self.assertSitemapBody(r, self.BODY) + # .xml.gz but body decoded by HttpCompression middleware already + r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_urls_from_robotstxt(self): robots = b"""# Sitemap files Sitemap: http://example.com/sitemap.xml