From 11cdf58abe26a9f65d8c1aefa717b6735e5734c9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 9 Nov 2016 23:08:23 +0100 Subject: [PATCH 1/4] Always decompress Content-Encoding: gzip at HttpCompression stage Let SitemapSpider handle decoding of .xml.gz files if necessary --- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/spiders/sitemap.py | 23 +++++-- scrapy/utils/gz.py | 4 ++ ...st_downloadermiddleware_httpcompression.py | 61 ++++++++++++++++--- tests/test_spider.py | 4 ++ 5 files changed, 78 insertions(+), 16 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index eb00d8923..dd32c62de 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -34,7 +34,7 @@ class HttpCompressionMiddleware(object): return response if isinstance(response, Response): content_encoding = response.headers.getlist('Content-Encoding') - if content_encoding and not is_gzipped(response): + if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) respcls = responsetypes.from_args(headers=response.headers, \ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 9e45637c3..10af90259 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -5,7 +5,8 @@ import six from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots -from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.utils.gz import gunzip, gzip_magic_number + logger = logging.getLogger(__name__) @@ -59,12 +60,22 @@ class SitemapSpider(Spider): """ if isinstance(response, XmlResponse): return response.body - elif is_gzipped(response): - return gunzip(response.body) - elif response.url.endswith('.xml'): + elif gzip_magic_number(response): + try: + return gunzip(response.body) + except: + pass + # actual gzipped sitemap files are decompressed above ; + # if we are here (response body is not gzipped) + # and have a response for .xml.gz, + # it usually means that it was already gunzipped + # by HttpCompression middleware, + # the HTTP response being sent with "Content-Encoding: gzip" + # without actually being a .xml.gz file in the first place, + # merely XML gzip-compressed on the fly, + # in other word, here, we have plain XML + elif response.url.endswith('.xml') or response.url.endswith('.xml.gz'): return response.body - elif response.url.endswith('.xml.gz'): - return gunzip(response.body) def regex(x): diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 73c2eb73b..22cf58986 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -59,3 +59,7 @@ def is_gzipped(response): cenc = response.headers.get('Content-Encoding', b'').lower() return (_is_gzipped(ctype) or (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip'))) + + +def gzip_magic_number(response): + return response.body[:2] == b'\x1f\x8b' diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5403e8f52..5f56c99ec 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -8,6 +8,7 @@ from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ ACCEPTED_ENCODINGS from scrapy.responsetypes import responsetypes +from scrapy.utils.gz import gunzip from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -173,9 +174,9 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) - self.assertIs(newresponse, response) - self.assertEqual(response.headers['Content-Encoding'], b'gzip') - self.assertEqual(response.headers['Content-Type'], b'application/gzip') + assert newresponse is not response + assert newresponse.body.startswith(b' + + + http://www.example.com/ + 2009-08-16 + daily + 1 + + + http://www.example.com/Special-Offers.html + 2009-08-16 + weekly + 0.8 + +""" + gz_file = GzipFile(fileobj=f, mode='wb') + gz_file.write(plainbody) + gz_file.close() + + # build a gzipped response body containing this gzipped file + r = BytesIO() + gz_resp = GzipFile(fileobj=r, mode='wb') + gz_resp.write(f.getvalue()) + gz_resp.close() + + response = Response("http;//www.example.com/", headers=headers, body=r.getvalue()) + request = Request("http://www.example.com/") + + newresponse = self.mw.process_response(request, response, self.spider) + self.assertEqual(gunzip(newresponse.body), plainbody) def test_process_response_head_request_no_decode_required(self): response = self._getresponse('gzip') diff --git a/tests/test_spider.py b/tests/test_spider.py index 371b8c1ac..e55f0fa6d 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -328,6 +328,10 @@ class SitemapSpiderTest(SpiderTest): r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) self.assertSitemapBody(r, self.BODY) + # .xml.gz but body decoded by HttpCompression middleware already + r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_urls_from_robotstxt(self): robots = b"""# Sitemap files Sitemap: http://example.com/sitemap.xml From b174744b80fd12efc6e4008ea5eb38256f21038a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 10 Nov 2016 10:50:34 +0100 Subject: [PATCH 2/4] Do not silently fail on gzip unzipping --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- scrapy/spiders/sitemap.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index dd32c62de..203dee42d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,6 @@ import zlib -from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.exceptions import NotConfigured diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 10af90259..e54001d88 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -61,10 +61,7 @@ class SitemapSpider(Spider): if isinstance(response, XmlResponse): return response.body elif gzip_magic_number(response): - try: - return gunzip(response.body) - except: - pass + return gunzip(response.body) # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) # and have a response for .xml.gz, From 4caceccd594f2563a3d32ea708579ce8e70132f3 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 10:51:34 +0100 Subject: [PATCH 3/4] Use 3-bytes for gzip archive type sniffing --- scrapy/utils/gz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 22cf58986..16c9ce539 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -62,4 +62,4 @@ def is_gzipped(response): def gzip_magic_number(response): - return response.body[:2] == b'\x1f\x8b' + return response.body[:3] == b'\x1f\x8b\x08' From b6378c7ef6393412165239fd6bf489d45a1c5196 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 12:28:24 +0100 Subject: [PATCH 4/4] Revert to using self.assert methods --- ...est_downloadermiddleware_httpcompression.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5f56c99ec..0678fcb14 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -174,9 +174,9 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) - assert newresponse is not response - assert newresponse.body.startswith(b'