RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Also use DOWNLOAD_WARNSIZE for decompressions

This commit is contained in:
Adrián Chaves 2023-11-23 10:26:47 +01:00
parent 1087bb7b2e
commit 03d9866518
6 changed files with 260 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -1,4 +1,5 @@
import warnings
from logging import getLogger
from scrapy import signals
from scrapy.exceptions import IgnoreRequest, NotConfigured
@ -13,6 +14,8 @@ from scrapy.utils._compression import (
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
logger = getLogger(__name__)
ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
try:
@ -39,6 +42,7 @@ class HttpCompressionMiddleware:
return
self.stats = crawler.stats
self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
crawler.signals.connect(self.open_spider, signals.spider_opened)
@classmethod
@ -57,12 +61,15 @@ class HttpCompressionMiddleware:
spider = cls()
spider.stats = crawler.stats
spider._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
spider._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
crawler.signals.connect(spider.open_spider, signals.spider_opened)
return spider
def open_spider(self, spider):
if hasattr(spider, "download_maxsize"):
self._max_size = spider.download_maxsize
if hasattr(spider, "download_warnsize"):
self._warn_size = spider.download_warnsize
def process_request(self, request, spider):
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
@ -75,6 +82,7 @@ class HttpCompressionMiddleware:
if content_encoding:
encoding = content_encoding.pop()
max_size = request.meta.get("download_maxsize", self._max_size)
warn_size = request.meta.get("download_warnsize", self._warn_size)
try:
decoded_body = self._decode(
response.body, encoding.lower(), max_size
@ -82,8 +90,14 @@ class HttpCompressionMiddleware:
except _DecompressionMaxSizeExceeded:
raise IgnoreRequest(
f"Ignored response {response} because its body "
f"({len(response.body)}B) exceeded DOWNLOAD_MAXSIZE "
f"({self._max_size}B) during decompression."
f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE "
f"({self._max_size} B) during decompression."
)
if len(response.body) < warn_size and len(decoded_body) >= warn_size:
logger.warning(
f"{response} body size after decompression "
f"({len(decoded_body)} B) is larger than the "
f"download warning size ({warn_size} B)."
)
if self.stats:
self.stats.inc_value(

35
scrapy/spiders/sitemap.py
View File

@ -1,5 +1,6 @@
import logging
import re
from typing import TYPE_CHECKING, Any
from scrapy.http import Request, XmlResponse
from scrapy.spiders import Spider
@ -7,6 +8,12 @@ from scrapy.utils._compression import _DecompressionMaxSizeExceeded
from scrapy.utils.gz import gunzip, gzip_magic_number
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
@ -16,6 +23,17 @@ class SitemapSpider(Spider):
sitemap_follow = [""]
sitemap_alternate_links = False
@classmethod
def from_crawler(cls, crawler: "Crawler", *args: Any, **kwargs: Any) -> "Self":
spider = super().from_crawler(crawler, *args, **kwargs)
spider._max_size = getattr(
spider, "download_maxsize", spider.settings.getint("DOWNLOAD_MAXSIZE")
)
spider._warn_size = getattr(
spider, "download_warnsize", spider.settings.getint("DOWNLOAD_WARNSIZE")
)
return spider
def __init__(self, *a, **kw):
super().__init__(*a, **kw)
self._cbs = []
@ -72,16 +90,19 @@ class SitemapSpider(Spider):
if isinstance(response, XmlResponse):
return response.body
if gzip_magic_number(response):
max_size = response.meta.get(
"download_maxsize",
getattr(
self, "download_maxsize", self.settings.getint("DOWNLOAD_MAXSIZE")
),
)
uncompressed_size = len(response.body)
max_size = response.meta.get("download_maxsize", self._max_size)
warn_size = response.meta.get("download_warnsize", self._warn_size)
try:
return gunzip(response.body, max_size=max_size)
body = gunzip(response.body, max_size=max_size)
except _DecompressionMaxSizeExceeded:
return None
if uncompressed_size < warn_size and len(body) >= warn_size:
logger.warning(
f"{response} body size after decompression ({len(body)} B) "
f"is larger than the download warning size ({warn_size} B)."
)
return body
# actual gzipped sitemap files are decompressed above ;
# if we are here (response body is not gzipped)
# and have a response for .xml.gz,

12
scrapy/utils/_compression.py
View File

@ -44,8 +44,8 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes:
if max_size and decompressed_size > max_size:
raise _DecompressionMaxSizeExceeded(
f"The number of bytes decompressed so far "
f"({decompressed_size}B) exceed the specified maximum "
f"({max_size}B)."
f"({decompressed_size} B) exceed the specified maximum "
f"({max_size} B)."
)
output_list.append(output_chunk)
return b"".join(output_list)
@ -65,8 +65,8 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
if max_size and decompressed_size > max_size:
raise _DecompressionMaxSizeExceeded(
f"The number of bytes decompressed so far "
f"({decompressed_size}B) exceed the specified maximum "
f"({max_size}B)."
f"({decompressed_size} B) exceed the specified maximum "
f"({max_size} B)."
)
output_list.append(output_chunk)
return b"".join(output_list)
@ -85,8 +85,8 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes:
if max_size and decompressed_size > max_size:
raise _DecompressionMaxSizeExceeded(
f"The number of bytes decompressed so far "
f"({decompressed_size}B) exceed the specified maximum "
f"({max_size}B)."
f"({decompressed_size} B) exceed the specified maximum "
f"({max_size} B)."
)
output_list.append(output_chunk)
return b"".join(output_list)

4
scrapy/utils/gz.py
View File

@ -31,8 +31,8 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes:
if max_size and decompressed_size > max_size:
raise _DecompressionMaxSizeExceeded(
f"The number of bytes decompressed so far "
f"({decompressed_size}B) exceed the specified maximum "
f"({max_size}B)."
f"({decompressed_size} B) exceed the specified maximum "
f"({max_size} B)."
)
output_list.append(chunk)
return b"".join(output_list)

130
tests/test_downloadermiddleware_httpcompression.py
View File

@ -1,9 +1,11 @@
from gzip import GzipFile
from io import BytesIO
from logging import WARNING
from pathlib import Path
from unittest import SkipTest, TestCase
from warnings import catch_warnings
from testfixtures import LogCapture
from w3lib.encoding import resolve_encoding
from scrapy.downloadermiddlewares.httpcompression import (
@ -468,6 +470,134 @@ class HttpCompressionTest(TestCase):
def test_compression_bomb_request_meta_zstd(self):
self._test_compression_bomb_request_meta("zstd")
def _test_download_warnsize_setting(self, compression_id):
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
with LogCapture(
"scrapy.downloadermiddlewares.httpcompression",
propagate=False,
level=WARNING,
) as log:
mw.process_response(response.request, response, spider)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
"WARNING",
(
"<200 http://scrapytest.org/> body size after "
"decompression (11511612 B) is larger than the download "
"warning size (10000000 B)."
),
),
)
def test_download_warnsize_setting_br(self):
try:
import brotli # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_download_warnsize_setting("br")
def test_download_warnsize_setting_deflate(self):
self._test_download_warnsize_setting("deflate")
def test_download_warnsize_setting_gzip(self):
self._test_download_warnsize_setting("gzip")
def test_download_warnsize_setting_zstd(self):
self._test_download_warnsize_setting("zstd")
def _test_download_warnsize_spider_attr(self, compression_id):
class DownloadWarnSizeSpider(Spider):
download_warnsize = 10_000_000
crawler = get_crawler(DownloadWarnSizeSpider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
with LogCapture(
"scrapy.downloadermiddlewares.httpcompression",
propagate=False,
level=WARNING,
) as log:
mw.process_response(response.request, response, spider)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
"WARNING",
(
"<200 http://scrapytest.org/> body size after "
"decompression (11511612 B) is larger than the download "
"warning size (10000000 B)."
),
),
)
def test_download_warnsize_spider_attr_br(self):
try:
import brotli # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_download_warnsize_spider_attr("br")
def test_download_warnsize_spider_attr_deflate(self):
self._test_download_warnsize_spider_attr("deflate")
def test_download_warnsize_spider_attr_gzip(self):
self._test_download_warnsize_spider_attr("gzip")
def test_download_warnsize_spider_attr_zstd(self):
self._test_download_warnsize_spider_attr("zstd")
def _test_download_warnsize_request_meta(self, compression_id):
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_warnsize"] = 10_000_000
with LogCapture(
"scrapy.downloadermiddlewares.httpcompression",
propagate=False,
level=WARNING,
) as log:
mw.process_response(response.request, response, spider)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
"WARNING",
(
"<200 http://scrapytest.org/> body size after "
"decompression (11511612 B) is larger than the download "
"warning size (10000000 B)."
),
),
)
def test_download_warnsize_request_meta_br(self):
try:
import brotli # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_download_warnsize_request_meta("br")
def test_download_warnsize_request_meta_deflate(self):
self._test_download_warnsize_request_meta("deflate")
def test_download_warnsize_request_meta_gzip(self):
self._test_download_warnsize_request_meta("gzip")
def test_download_warnsize_request_meta_zstd(self):
self._test_download_warnsize_request_meta("zstd")
class HttpCompressionSubclassTest(TestCase):
def test_init_missing_stats(self):

78
tests/test_spider.py
View File

@ -2,6 +2,7 @@ import gzip
import inspect
import warnings
from io import BytesIO
from logging import WARNING
from pathlib import Path
from typing import Any
from unittest import mock
@ -732,6 +733,83 @@ Sitemap: /sitemap-relative-url.xml
response = Response(url="https://example.com", body=body, request=request)
self.assertIsNone(spider._get_sitemap_body(response))
def test_download_warnsize_setting(self):
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
crawler = get_crawler(settings_dict=settings)
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
def test_download_warnsize_spider_attr(self):
class DownloadWarnSizeSpider(self.spider_class):
download_warnsize = 10_000_000
crawler = get_crawler()
spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(
url="https://example.com", meta={"download_warnsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
def test_download_warnsize_request_meta(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(
url="https://example.com", meta={"download_warnsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
class DeprecationTest(unittest.TestCase):
def test_crawl_spider(self):