Mind Spider.download_maxsize and Request.meta['download_maxsize']

scrapy/downloadermiddlewares/httpcompression.py

@@ -1,5 +1,6 @@
 import warnings
 
+from scrapy import signals
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
@@ -38,6 +39,7 @@ class HttpCompressionMiddleware:
             return
         self.stats = crawler.stats
         self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
+        crawler.signals.connect(self.open_spider, signals.spider_opened)
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -52,10 +54,15 @@ class HttpCompressionMiddleware:
                 "reimplement their 'from_crawler' method.",
                 ScrapyDeprecationWarning,
             )
-            result = cls()
+            spider = cls()
-            result.stats = crawler.stats
+            spider.stats = crawler.stats
-            result._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
+            spider._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
-            return result
+            crawler.signals.connect(spider.open_spider, signals.spider_opened)
+            return spider
+
+    def open_spider(self, spider):
+        if hasattr(spider, "download_maxsize"):
+            self._max_size = spider.download_maxsize
 
     def process_request(self, request, spider):
         request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
@@ -67,8 +74,11 @@ class HttpCompressionMiddleware:
             content_encoding = response.headers.getlist("Content-Encoding")
             if content_encoding:
                 encoding = content_encoding.pop()
+                max_size = request.meta.get("download_maxsize", self._max_size)
                 try:
-                    decoded_body = self._decode(response.body, encoding.lower())
+                    decoded_body = self._decode(
+                        response.body, encoding.lower(), max_size
+                    )
                 except _DecompressionMaxSizeExceeded:
                     raise IgnoreRequest(
                         f"Ignored response {response} because its body "
@@ -98,13 +108,13 @@ class HttpCompressionMiddleware:
 
         return response
 
-    def _decode(self, body, encoding):
+    def _decode(self, body, encoding, max_size):
         if encoding == b"gzip" or encoding == b"x-gzip":
-            return gunzip(body, max_size=self._max_size)
+            return gunzip(body, max_size=max_size)
         if encoding == b"deflate":
-            return _inflate(body, max_size=self._max_size)
+            return _inflate(body, max_size=max_size)
         if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
-            return _unbrotli(body, max_size=self._max_size)
+            return _unbrotli(body, max_size=max_size)
         if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
-            return _unzstd(body, max_size=self._max_size)
+            return _unzstd(body, max_size=max_size)
         return body

scrapy/spiders/sitemap.py

@@ -72,10 +72,14 @@ class SitemapSpider(Spider):
         if isinstance(response, XmlResponse):
             return response.body
         if gzip_magic_number(response):
+            max_size = response.meta.get(
+                "download_maxsize",
+                getattr(
+                    self, "download_maxsize", self.settings.getint("DOWNLOAD_MAXSIZE")
+                ),
+            )
             try:
-                return gunzip(
+                return gunzip(response.body, max_size=max_size)
-                    response.body, max_size=self.settings.getint("DOWNLOAD_MAXSIZE")
-                )
             except _DecompressionMaxSizeExceeded:
                 return None
         # actual gzipped sitemap files are decompressed above ;

tests/test_downloadermiddleware_httpcompression.py

@@ -49,10 +49,7 @@ FORMAT = {
 
 class HttpCompressionTest(TestCase):
     def setUp(self):
-        settings = {
+        self.crawler = get_crawler(Spider)
-            "DOWNLOAD_MAXSIZE": 10_000_000,  # For compression bomb tests.
-        }
-        self.crawler = get_crawler(Spider, settings_dict=settings)
         self.spider = self.crawler._create_spider("scrapytest.org")
         self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
         self.crawler.stats.open_spider(self.spider)
@@ -373,31 +370,103 @@ class HttpCompressionTest(TestCase):
         self.assertStatsEqual("httpcompression/response_count", None)
         self.assertStatsEqual("httpcompression/response_bytes", None)
 
-    def _test_compression_bomb(self, compression_id):
+    def _test_compression_bomb_setting(self, compression_id):
+        settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
+        crawler = get_crawler(Spider, settings_dict=settings)
+        spider = crawler._create_spider("scrapytest.org")
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        mw.open_spider(spider)
+
         response = self._getresponse(f"bomb-{compression_id}")
         self.assertRaises(
             IgnoreRequest,
-            self.mw.process_response,
+            mw.process_response,
             response.request,
             response,
-            self.spider,
+            spider,
         )
 
-    def test_compression_bomb_br(self):
+    def test_compression_bomb_setting_br(self):
         try:
             import brotli  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
-        self._test_compression_bomb("br")
+        self._test_compression_bomb_setting("br")
 
-    def test_compression_bomb_deflate(self):
+    def test_compression_bomb_setting_deflate(self):
-        self._test_compression_bomb("deflate")
+        self._test_compression_bomb_setting("deflate")
 
-    def test_compression_bomb_gzip(self):
+    def test_compression_bomb_setting_gzip(self):
-        self._test_compression_bomb("gzip")
+        self._test_compression_bomb_setting("gzip")
 
-    def test_compression_bomb_zstd(self):
+    def test_compression_bomb_setting_zstd(self):
-        self._test_compression_bomb("zstd")
+        self._test_compression_bomb_setting("zstd")
+
+    def _test_compression_bomb_spider_attr(self, compression_id):
+        class DownloadMaxSizeSpider(Spider):
+            download_maxsize = 10_000_000
+
+        crawler = get_crawler(DownloadMaxSizeSpider)
+        spider = crawler._create_spider("scrapytest.org")
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        mw.open_spider(spider)
+
+        response = self._getresponse(f"bomb-{compression_id}")
+        self.assertRaises(
+            IgnoreRequest,
+            mw.process_response,
+            response.request,
+            response,
+            spider,
+        )
+
+    def test_compression_bomb_spider_attr_br(self):
+        try:
+            import brotli  # noqa: F401
+        except ImportError:
+            raise SkipTest("no brotli")
+        self._test_compression_bomb_spider_attr("br")
+
+    def test_compression_bomb_spider_attr_deflate(self):
+        self._test_compression_bomb_spider_attr("deflate")
+
+    def test_compression_bomb_spider_attr_gzip(self):
+        self._test_compression_bomb_spider_attr("gzip")
+
+    def test_compression_bomb_spider_attr_zstd(self):
+        self._test_compression_bomb_spider_attr("zstd")
+
+    def _test_compression_bomb_request_meta(self, compression_id):
+        crawler = get_crawler(Spider)
+        spider = crawler._create_spider("scrapytest.org")
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        mw.open_spider(spider)
+
+        response = self._getresponse(f"bomb-{compression_id}")
+        response.meta["download_maxsize"] = 10_000_000
+        self.assertRaises(
+            IgnoreRequest,
+            mw.process_response,
+            response.request,
+            response,
+            spider,
+        )
+
+    def test_compression_bomb_request_meta_br(self):
+        try:
+            import brotli  # noqa: F401
+        except ImportError:
+            raise SkipTest("no brotli")
+        self._test_compression_bomb_request_meta("br")
+
+    def test_compression_bomb_request_meta_deflate(self):
+        self._test_compression_bomb_request_meta("deflate")
+
+    def test_compression_bomb_request_meta_gzip(self):
+        self._test_compression_bomb_request_meta("gzip")
+
+    def test_compression_bomb_request_meta_zstd(self):
+        self._test_compression_bomb_request_meta("zstd")
 
 
 class HttpCompressionSubclassTest(TestCase):

tests/test_spider.py

@@ -509,6 +509,7 @@ class SitemapSpiderTest(SpiderTest):
             url="http://www.example.com/sitemap",
             body=self.GZBODY,
             headers={"content-type": "application/gzip"},
+            request=Request("http://www.example.com/sitemap"),
         )
         self.assertSitemapBody(r, self.BODY)
 
@@ -517,7 +518,11 @@ class SitemapSpiderTest(SpiderTest):
         self.assertSitemapBody(r, self.BODY)
 
     def test_get_sitemap_body_xml_url_compressed(self):
-        r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY)
+        r = Response(
+            url="http://www.example.com/sitemap.xml.gz",
+            body=self.GZBODY,
+            request=Request("http://www.example.com/sitemap"),
+        )
         self.assertSitemapBody(r, self.BODY)
 
         # .xml.gz but body decoded by HttpCompression middleware already
@@ -694,13 +699,37 @@ Sitemap: /sitemap-relative-url.xml
             ["http://www.example.com/sitemap2.xml"],
         )
 
-    def test_compression_bomb(self):
+    def test_compression_bomb_setting(self):
         settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
         crawler = get_crawler(settings_dict=settings)
         spider = self.spider_class.from_crawler(crawler, "example.com")
         body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
         body = body_path.read_bytes()
-        response = Response(url="https://example.com", body=body)
+        request = Request(url="https://example.com")
+        response = Response(url="https://example.com", body=body, request=request)
+        self.assertIsNone(spider._get_sitemap_body(response))
+
+    def test_compression_bomb_spider_attr(self):
+        class DownloadMaxSizeSpider(self.spider_class):
+            download_maxsize = 10_000_000
+
+        crawler = get_crawler()
+        spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com")
+        body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
+        body = body_path.read_bytes()
+        request = Request(url="https://example.com")
+        response = Response(url="https://example.com", body=body, request=request)
+        self.assertIsNone(spider._get_sitemap_body(response))
+
+    def test_compression_bomb_request_meta(self):
+        crawler = get_crawler()
+        spider = self.spider_class.from_crawler(crawler, "example.com")
+        body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
+        body = body_path.read_bytes()
+        request = Request(
+            url="https://example.com", meta={"download_maxsize": 10_000_000}
+        )
+        response = Response(url="https://example.com", body=body, request=request)
         self.assertIsNone(spider._get_sitemap_body(response))