RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch '2.11-compression-bomb' into 2.11

This commit is contained in:
Adrián Chaves 2024-02-14 17:16:25 +01:00
parent 5bcb8fd501 12b10a7a64
commit 809bfac489
15 changed files with 613 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
docs/news.rst
View File

@ -19,6 +19,16 @@ Highlights:
Security bug fixes
~~~~~~~~~~~~~~~~~~
- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply
to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security
advisory`_ for more information.
.. _7j7m-v7m3-jqm7 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-7j7m-v7m3-jqm7
- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, the
deprecated ``scrapy.downloadermiddlewares.decompression`` module has been
removed.
- The ``Authorization`` header is now dropped on redirects to a different
domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more
information.
@ -2941,6 +2951,7 @@ affect subclasses:
(:issue:`3884`)
.. _release-1.8.4:
Scrapy 1.8.4 (unreleased)
@ -2948,6 +2959,14 @@ Scrapy 1.8.4 (unreleased)
**Security bug fixes:**
- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply
to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security
advisory`_ for more information.
- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, use of the
``scrapy.downloadermiddlewares.decompression`` module is discouraged and
will trigger a warning.
- The ``Authorization`` header is now dropped on redirects to a different
domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more
information.

1
docs/topics/request-response.rst
View File

@ -731,6 +731,7 @@ Those are:
* :reqmeta:`download_fail_on_dataloss`
* :reqmeta:`download_latency`
* :reqmeta:`download_maxsize`
* :reqmeta:`download_warnsize`
* :reqmeta:`download_timeout`
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
* ``ftp_user`` (See :setting:`FTP_USER` for more info)

36
docs/topics/settings.rst
View File

@ -873,40 +873,42 @@ The amount of time (in secs) that the downloader will wait before timing out.
Request.meta key.
.. setting:: DOWNLOAD_MAXSIZE
.. reqmeta:: download_maxsize
DOWNLOAD_MAXSIZE
----------------
Default: ``1073741824`` (1024MB)
Default: ``1073741824`` (1 GiB)
The maximum response size (in bytes) that downloader will download.
The maximum response body size (in bytes) allowed. Bigger responses are
aborted and ignored.
If you want to disable it set to 0.
This applies both before and after compression. If decompressing a response
body would exceed this limit, decompression is aborted and the response is
ignored.
.. reqmeta:: download_maxsize
Use ``0`` to disable this limit.
.. note::
This size can be set per spider using :attr:`download_maxsize`
spider attribute and per-request using :reqmeta:`download_maxsize`
Request.meta key.
This limit can be set per spider using the :attr:`download_maxsize` spider
attribute and per request using the :reqmeta:`download_maxsize` Request.meta
key.
.. setting:: DOWNLOAD_WARNSIZE
.. reqmeta:: download_warnsize
DOWNLOAD_WARNSIZE
-----------------
Default: ``33554432`` (32MB)
Default: ``33554432`` (32 MiB)
The response size (in bytes) that downloader will start to warn.
If the size of a response exceeds this value, before or after compression, a
warning will be logged about it.
If you want to disable it set to 0.
Use ``0`` to disable this limit.
.. note::
This size can be set per spider using :attr:`download_warnsize`
spider attribute and per-request using :reqmeta:`download_warnsize`
Request.meta key.
This limit can be set per spider using the :attr:`download_warnsize` spider
attribute and per request using the :reqmeta:`download_warnsize` Request.meta
key.
.. setting:: DOWNLOAD_FAIL_ON_DATALOSS

94
scrapy/downloadermiddlewares/decompression.py
View File

@ -1,94 +0,0 @@
""" This module implements the DecompressionMiddleware which tries to recognise
and extract the potentially compressed responses that may arrive.
"""
import bz2
import gzip
import logging
import tarfile
import zipfile
from io import BytesIO
from tempfile import mktemp
from warnings import warn
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.responsetypes import responsetypes
warn(
"scrapy.downloadermiddlewares.decompression is deprecated",
ScrapyDeprecationWarning,
stacklevel=2,
)
logger = logging.getLogger(__name__)
class DecompressionMiddleware:
"""This middleware tries to recognise and extract the possibly compressed
responses that may arrive."""
def __init__(self):
self._formats = {
"tar": self._is_tar,
"zip": self._is_zip,
"gz": self._is_gzip,
"bz2": self._is_bzip2,
}
def _is_tar(self, response):
archive = BytesIO(response.body)
try:
tar_file = tarfile.open(name=mktemp(), fileobj=archive)
except tarfile.ReadError:
return
body = tar_file.extractfile(tar_file.members[0]).read()
respcls = responsetypes.from_args(filename=tar_file.members[0].name, body=body)
return response.replace(body=body, cls=respcls)
def _is_zip(self, response):
archive = BytesIO(response.body)
try:
zip_file = zipfile.ZipFile(archive)
except zipfile.BadZipFile:
return
namelist = zip_file.namelist()
body = zip_file.read(namelist[0])
respcls = responsetypes.from_args(filename=namelist[0], body=body)
return response.replace(body=body, cls=respcls)
def _is_gzip(self, response):
archive = BytesIO(response.body)
try:
body = gzip.GzipFile(fileobj=archive).read()
except OSError:
return
respcls = responsetypes.from_args(body=body)
return response.replace(body=body, cls=respcls)
def _is_bzip2(self, response):
try:
body = bz2.decompress(response.body)
except OSError:
return
respcls = responsetypes.from_args(body=body)
return response.replace(body=body, cls=respcls)
def process_response(self, request, response, spider):
if not response.body:
return response
for fmt, func in self._formats.items():
new_response = func(response)
if new_response:
logger.debug(
"Decompressed response with format: %(responsefmt)s",
{"responsefmt": fmt},
extra={"spider": spider},
)
return new_response
return response

100
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -1,53 +1,78 @@
import io
import warnings
import zlib
from logging import getLogger
from scrapy.exceptions import NotConfigured
from scrapy import signals
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils._compression import (
_DecompressionMaxSizeExceeded,
_inflate,
_unbrotli,
_unzstd,
)
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
logger = getLogger(__name__)
ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
try:
import brotli
ACCEPTED_ENCODINGS.append(b"br")
import brotli # noqa: F401
except ImportError:
pass
else:
ACCEPTED_ENCODINGS.append(b"br")
try:
import zstandard
ACCEPTED_ENCODINGS.append(b"zstd")
import zstandard # noqa: F401
except ImportError:
pass
else:
ACCEPTED_ENCODINGS.append(b"zstd")
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
def __init__(self, stats=None):
self.stats = stats
def __init__(self, stats=None, *, crawler=None):
if not crawler:
self.stats = stats
self._max_size = 1073741824
self._warn_size = 33554432
return
self.stats = crawler.stats
self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
crawler.signals.connect(self.open_spider, signals.spider_opened)
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
raise NotConfigured
try:
return cls(stats=crawler.stats)
return cls(crawler=crawler)
except TypeError:
warnings.warn(
"HttpCompressionMiddleware subclasses must either modify "
"their '__init__' method to support a 'stats' parameter or "
"reimplement the 'from_crawler' method.",
"their '__init__' method to support a 'crawler' parameter or "
"reimplement their 'from_crawler' method.",
ScrapyDeprecationWarning,
)
result = cls()
result.stats = crawler.stats
return result
mw = cls()
mw.stats = crawler.stats
mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
crawler.signals.connect(mw.open_spider, signals.spider_opened)
return mw
def open_spider(self, spider):
if hasattr(spider, "download_maxsize"):
self._max_size = spider.download_maxsize
if hasattr(spider, "download_warnsize"):
self._warn_size = spider.download_warnsize
def process_request(self, request, spider):
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
@ -59,7 +84,24 @@ class HttpCompressionMiddleware:
content_encoding = response.headers.getlist("Content-Encoding")
if content_encoding:
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
max_size = request.meta.get("download_maxsize", self._max_size)
warn_size = request.meta.get("download_warnsize", self._warn_size)
try:
decoded_body = self._decode(
response.body, encoding.lower(), max_size
)
except _DecompressionMaxSizeExceeded:
raise IgnoreRequest(
f"Ignored response {response} because its body "
f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE "
f"({max_size} B) during decompression."
)
if len(response.body) < warn_size <= len(decoded_body):
logger.warning(
f"{response} body size after decompression "
f"({len(decoded_body)} B) is larger than the "
f"download warning size ({warn_size} B)."
)
if self.stats:
self.stats.inc_value(
"httpcompression/response_bytes",
@ -83,25 +125,13 @@ class HttpCompressionMiddleware:
return response
def _decode(self, body, encoding):
def _decode(self, body, encoding, max_size):
if encoding == b"gzip" or encoding == b"x-gzip":
body = gunzip(body)
return gunzip(body, max_size=max_size)
if encoding == b"deflate":
try:
body = zlib.decompress(body)
except zlib.error:
# ugly hack to work with raw deflate content that may
# be sent by microsoft servers. For more information, see:
# http://carsten.codimi.de/gzip.yaws/
# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
# http://www.gzip.org/zlib/zlib_faq.html#faq38
body = zlib.decompress(body, -15)
return _inflate(body, max_size=max_size)
if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
body = brotli.decompress(body)
return _unbrotli(body, max_size=max_size)
if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
# Using its streaming API since its simple API could handle only cases
# where there is content size data embedded in the frame
reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
body = reader.read()
return _unzstd(body, max_size=max_size)
return body

35
scrapy/spiders/sitemap.py
View File

@ -1,11 +1,19 @@
import logging
import re
from typing import TYPE_CHECKING, Any
from scrapy.http import Request, XmlResponse
from scrapy.spiders import Spider
from scrapy.utils._compression import _DecompressionMaxSizeExceeded
from scrapy.utils.gz import gunzip, gzip_magic_number
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
@ -14,6 +22,19 @@ class SitemapSpider(Spider):
sitemap_rules = [("", "parse")]
sitemap_follow = [""]
sitemap_alternate_links = False
_max_size: int
_warn_size: int
@classmethod
def from_crawler(cls, crawler: "Crawler", *args: Any, **kwargs: Any) -> "Self":
spider = super().from_crawler(crawler, *args, **kwargs)
spider._max_size = getattr(
spider, "download_maxsize", spider.settings.getint("DOWNLOAD_MAXSIZE")
)
spider._warn_size = getattr(
spider, "download_warnsize", spider.settings.getint("DOWNLOAD_WARNSIZE")
)
return spider
def __init__(self, *a, **kw):
super().__init__(*a, **kw)
@ -71,7 +92,19 @@ class SitemapSpider(Spider):
if isinstance(response, XmlResponse):
return response.body
if gzip_magic_number(response):
return gunzip(response.body)
uncompressed_size = len(response.body)
max_size = response.meta.get("download_maxsize", self._max_size)
warn_size = response.meta.get("download_warnsize", self._warn_size)
try:
body = gunzip(response.body, max_size=max_size)
except _DecompressionMaxSizeExceeded:
return None
if uncompressed_size < warn_size <= len(body):
logger.warning(
f"{response} body size after decompression ({len(body)} B) "
f"is larger than the download warning size ({warn_size} B)."
)
return body
# actual gzipped sitemap files are decompressed above ;
# if we are here (response body is not gzipped)
# and have a response for .xml.gz,

94
scrapy/utils/_compression.py Normal file
View File

@ -0,0 +1,94 @@
import zlib
from io import BytesIO
try:
import brotli
except ImportError:
pass
try:
import zstandard
except ImportError:
pass
_CHUNK_SIZE = 65536 # 64 KiB
class _DecompressionMaxSizeExceeded(ValueError):
pass
def _inflate(data: bytes, *, max_size: int = 0) -> bytes:
decompressor = zlib.decompressobj()
raw_decompressor = zlib.decompressobj(wbits=-15)
input_stream = BytesIO(data)
output_stream = BytesIO()
output_chunk = b"."
decompressed_size = 0
while output_chunk:
input_chunk = input_stream.read(_CHUNK_SIZE)
try:
output_chunk = decompressor.decompress(input_chunk)
except zlib.error:
if decompressor != raw_decompressor:
# ugly hack to work with raw deflate content that may
# be sent by microsoft servers. For more information, see:
# http://carsten.codimi.de/gzip.yaws/
# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
# http://www.gzip.org/zlib/zlib_faq.html#faq38
decompressor = raw_decompressor
output_chunk = decompressor.decompress(input_chunk)
else:
raise
decompressed_size += len(output_chunk)
if max_size and decompressed_size > max_size:
raise _DecompressionMaxSizeExceeded(
f"The number of bytes decompressed so far "
f"({decompressed_size} B) exceed the specified maximum "
f"({max_size} B)."
)
output_stream.write(output_chunk)
output_stream.seek(0)
return output_stream.read()
def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
decompressor = brotli.Decompressor()
input_stream = BytesIO(data)
output_stream = BytesIO()
output_chunk = b"."
decompressed_size = 0
while output_chunk:
input_chunk = input_stream.read(_CHUNK_SIZE)
output_chunk = decompressor.process(input_chunk)
decompressed_size += len(output_chunk)
if max_size and decompressed_size > max_size:
raise _DecompressionMaxSizeExceeded(
f"The number of bytes decompressed so far "
f"({decompressed_size} B) exceed the specified maximum "
f"({max_size} B)."
)
output_stream.write(output_chunk)
output_stream.seek(0)
return output_stream.read()
def _unzstd(data: bytes, *, max_size: int = 0) -> bytes:
decompressor = zstandard.ZstdDecompressor()
stream_reader = decompressor.stream_reader(BytesIO(data))
output_stream = BytesIO()
output_chunk = b"."
decompressed_size = 0
while output_chunk:
output_chunk = stream_reader.read(_CHUNK_SIZE)
decompressed_size += len(output_chunk)
if max_size and decompressed_size > max_size:
raise _DecompressionMaxSizeExceeded(
f"The number of bytes decompressed so far "
f"({decompressed_size} B) exceed the specified maximum "
f"({max_size} B)."
)
output_stream.write(output_chunk)
output_stream.seek(0)
return output_stream.read()

26
scrapy/utils/gz.py
View File

@ -1,31 +1,41 @@
import struct
from gzip import GzipFile
from io import BytesIO
from typing import List
from scrapy.http import Response
from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded
def gunzip(data: bytes) -> bytes:
def gunzip(data: bytes, *, max_size: int = 0) -> bytes:
"""Gunzip the given data and return as much data as possible.
This is resilient to CRC checksum errors.
"""
f = GzipFile(fileobj=BytesIO(data))
output_list: List[bytes] = []
output_stream = BytesIO()
chunk = b"."
decompressed_size = 0
while chunk:
try:
chunk = f.read1(8196)
output_list.append(chunk)
chunk = f.read1(_CHUNK_SIZE)
except (OSError, EOFError, struct.error):
# complete only if there is some data, otherwise re-raise
# see issue 87 about catching struct.error
# some pages are quite small so output_list is empty
if output_list:
# some pages are quite small so output_stream is empty
if output_stream.getbuffer().nbytes > 0:
break
raise
return b"".join(output_list)
decompressed_size += len(chunk)
if max_size and decompressed_size > max_size:
raise _DecompressionMaxSizeExceeded(
f"The number of bytes decompressed so far "
f"({decompressed_size} B) exceed the specified maximum "
f"({max_size} B)."
)
output_stream.write(chunk)
output_stream.seek(0)
return output_stream.read()
def gzip_magic_number(response: Response) -> bool:

2
tests/sample_data/compressed/bomb-br.bin Normal file
View File

@ -0,0 +1,2 @@
<EFBFBD>;§¯ר<C2AF>”n<E2809D>×Vp SmoY2·ו
ן(ה)-׀´=_o

BIN
tests/sample_data/compressed/bomb-deflate.bin Normal file
View File

Binary file not shown.

BIN
tests/sample_data/compressed/bomb-gzip.bin Normal file
View File

Binary file not shown.

BIN
tests/sample_data/compressed/bomb-zstd.bin Normal file
View File

Binary file not shown.

53
tests/test_downloadermiddleware_decompression.py
View File

@ -1,53 +0,0 @@
from unittest import TestCase, main
from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware
from scrapy.http import Response, XmlResponse
from scrapy.spiders import Spider
from scrapy.utils.test import assert_samelines
from tests import get_testdata
def _test_data(formats):
uncompressed_body = get_testdata("compressed", "feed-sample1.xml")
test_responses = {}
for format in formats:
body = get_testdata("compressed", "feed-sample1." + format)
test_responses[format] = Response("http://foo.com/bar", body=body)
return uncompressed_body, test_responses
class DecompressionMiddlewareTest(TestCase):
test_formats = ["tar", "xml.bz2", "xml.gz", "zip"]
uncompressed_body, test_responses = _test_data(test_formats)
def setUp(self):
self.mw = DecompressionMiddleware()
self.spider = Spider("foo")
def test_known_compression_formats(self):
for fmt in self.test_formats:
rsp = self.test_responses[fmt]
new = self.mw.process_response(None, rsp, self.spider)
error_msg = f"Failed {fmt}, response type {type(new).__name__}"
assert isinstance(new, XmlResponse), error_msg
assert_samelines(self, new.body, self.uncompressed_body, fmt)
def test_plain_response(self):
rsp = Response(url="http://test.com", body=self.uncompressed_body)
new = self.mw.process_response(None, rsp, self.spider)
assert new is rsp
assert_samelines(self, new.body, rsp.body)
def test_empty_response(self):
rsp = Response(url="http://test.com", body=b"")
new = self.mw.process_response(None, rsp, self.spider)
assert new is rsp
assert not rsp.body
assert not new.body
def tearDown(self):
del self.mw
if __name__ == "__main__":
main()

255
tests/test_downloadermiddleware_httpcompression.py
View File

@ -1,16 +1,18 @@
from gzip import GzipFile
from io import BytesIO
from logging import WARNING
from pathlib import Path
from unittest import SkipTest, TestCase
from warnings import catch_warnings
from testfixtures import LogCapture
from w3lib.encoding import resolve_encoding
from scrapy.downloadermiddlewares.httpcompression import (
ACCEPTED_ENCODINGS,
HttpCompressionMiddleware,
)
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
@ -35,6 +37,15 @@ FORMAT = {
"html-zstd-streaming-no-content-size.bin",
"zstd",
),
**{
f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id)
for format_id in (
"br", # 34 → 11 511 612
"deflate", # 27 968 → 11 511 612
"gzip", # 27 988 → 11 511 612
"zstd", # 1 096 → 11 511 612
)
},
}
@ -115,18 +126,6 @@ class HttpCompressionTest(TestCase):
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
def test_process_response_gzip_no_stats(self):
mw = HttpCompressionMiddleware()
response = self._getresponse("gzip")
request = response.request
self.assertEqual(response.headers["Content-Encoding"], b"gzip")
newresponse = mw.process_response(request, response, self.spider)
self.assertEqual(mw.stats, None)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
def test_process_response_br(self):
try:
import brotli # noqa: F401
@ -373,6 +372,232 @@ class HttpCompressionTest(TestCase):
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
def _test_compression_bomb_setting(self, compression_id):
settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
self.assertRaises(
IgnoreRequest,
mw.process_response,
response.request,
response,
spider,
)
def test_compression_bomb_setting_br(self):
try:
import brotli # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_compression_bomb_setting("br")
def test_compression_bomb_setting_deflate(self):
self._test_compression_bomb_setting("deflate")
def test_compression_bomb_setting_gzip(self):
self._test_compression_bomb_setting("gzip")
def test_compression_bomb_setting_zstd(self):
self._test_compression_bomb_setting("zstd")
def _test_compression_bomb_spider_attr(self, compression_id):
class DownloadMaxSizeSpider(Spider):
download_maxsize = 10_000_000
crawler = get_crawler(DownloadMaxSizeSpider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
self.assertRaises(
IgnoreRequest,
mw.process_response,
response.request,
response,
spider,
)
def test_compression_bomb_spider_attr_br(self):
try:
import brotli # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_compression_bomb_spider_attr("br")
def test_compression_bomb_spider_attr_deflate(self):
self._test_compression_bomb_spider_attr("deflate")
def test_compression_bomb_spider_attr_gzip(self):
self._test_compression_bomb_spider_attr("gzip")
def test_compression_bomb_spider_attr_zstd(self):
self._test_compression_bomb_spider_attr("zstd")
def _test_compression_bomb_request_meta(self, compression_id):
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_maxsize"] = 10_000_000
self.assertRaises(
IgnoreRequest,
mw.process_response,
response.request,
response,
spider,
)
def test_compression_bomb_request_meta_br(self):
try:
import brotli # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_compression_bomb_request_meta("br")
def test_compression_bomb_request_meta_deflate(self):
self._test_compression_bomb_request_meta("deflate")
def test_compression_bomb_request_meta_gzip(self):
self._test_compression_bomb_request_meta("gzip")
def test_compression_bomb_request_meta_zstd(self):
self._test_compression_bomb_request_meta("zstd")
def _test_download_warnsize_setting(self, compression_id):
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
with LogCapture(
"scrapy.downloadermiddlewares.httpcompression",
propagate=False,
level=WARNING,
) as log:
mw.process_response(response.request, response, spider)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
"WARNING",
(
"<200 http://scrapytest.org/> body size after "
"decompression (11511612 B) is larger than the download "
"warning size (10000000 B)."
),
),
)
def test_download_warnsize_setting_br(self):
try:
import brotli # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_download_warnsize_setting("br")
def test_download_warnsize_setting_deflate(self):
self._test_download_warnsize_setting("deflate")
def test_download_warnsize_setting_gzip(self):
self._test_download_warnsize_setting("gzip")
def test_download_warnsize_setting_zstd(self):
self._test_download_warnsize_setting("zstd")
def _test_download_warnsize_spider_attr(self, compression_id):
class DownloadWarnSizeSpider(Spider):
download_warnsize = 10_000_000
crawler = get_crawler(DownloadWarnSizeSpider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
with LogCapture(
"scrapy.downloadermiddlewares.httpcompression",
propagate=False,
level=WARNING,
) as log:
mw.process_response(response.request, response, spider)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
"WARNING",
(
"<200 http://scrapytest.org/> body size after "
"decompression (11511612 B) is larger than the download "
"warning size (10000000 B)."
),
),
)
def test_download_warnsize_spider_attr_br(self):
try:
import brotli # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_download_warnsize_spider_attr("br")
def test_download_warnsize_spider_attr_deflate(self):
self._test_download_warnsize_spider_attr("deflate")
def test_download_warnsize_spider_attr_gzip(self):
self._test_download_warnsize_spider_attr("gzip")
def test_download_warnsize_spider_attr_zstd(self):
self._test_download_warnsize_spider_attr("zstd")
def _test_download_warnsize_request_meta(self, compression_id):
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_warnsize"] = 10_000_000
with LogCapture(
"scrapy.downloadermiddlewares.httpcompression",
propagate=False,
level=WARNING,
) as log:
mw.process_response(response.request, response, spider)
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
"WARNING",
(
"<200 http://scrapytest.org/> body size after "
"decompression (11511612 B) is larger than the download "
"warning size (10000000 B)."
),
),
)
def test_download_warnsize_request_meta_br(self):
try:
import brotli # noqa: F401
except ImportError:
raise SkipTest("no brotli")
self._test_download_warnsize_request_meta("br")
def test_download_warnsize_request_meta_deflate(self):
self._test_download_warnsize_request_meta("deflate")
def test_download_warnsize_request_meta_gzip(self):
self._test_download_warnsize_request_meta("gzip")
def test_download_warnsize_request_meta_zstd(self):
self._test_download_warnsize_request_meta("zstd")
class HttpCompressionSubclassTest(TestCase):
def test_init_missing_stats(self):
@ -393,8 +618,8 @@ class HttpCompressionSubclassTest(TestCase):
(
(
"HttpCompressionMiddleware subclasses must either modify "
"their '__init__' method to support a 'stats' parameter "
"or reimplement the 'from_crawler' method."
"their '__init__' method to support a 'crawler' parameter "
"or reimplement their 'from_crawler' method."
),
),
)

124
tests/test_spider.py
View File

@ -2,6 +2,8 @@ import gzip
import inspect
import warnings
from io import BytesIO
from logging import WARNING
from pathlib import Path
from typing import Any
from unittest import mock
@ -25,7 +27,7 @@ from scrapy.spiders import (
)
from scrapy.spiders.init import InitSpider
from scrapy.utils.test import get_crawler
from tests import get_testdata
from tests import get_testdata, tests_datadir
class SpiderTest(unittest.TestCase):
@ -489,7 +491,8 @@ class SitemapSpiderTest(SpiderTest):
GZBODY = f.getvalue()
def assertSitemapBody(self, response, body):
spider = self.spider_class("example.com")
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
self.assertEqual(spider._get_sitemap_body(response), body)
def test_get_sitemap_body(self):
@ -507,6 +510,7 @@ class SitemapSpiderTest(SpiderTest):
url="http://www.example.com/sitemap",
body=self.GZBODY,
headers={"content-type": "application/gzip"},
request=Request("http://www.example.com/sitemap"),
)
self.assertSitemapBody(r, self.BODY)
@ -515,7 +519,11 @@ class SitemapSpiderTest(SpiderTest):
self.assertSitemapBody(r, self.BODY)
def test_get_sitemap_body_xml_url_compressed(self):
r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY)
r = Response(
url="http://www.example.com/sitemap.xml.gz",
body=self.GZBODY,
request=Request("http://www.example.com/sitemap"),
)
self.assertSitemapBody(r, self.BODY)
# .xml.gz but body decoded by HttpCompression middleware already
@ -692,6 +700,116 @@ Sitemap: /sitemap-relative-url.xml
["http://www.example.com/sitemap2.xml"],
)
def test_compression_bomb_setting(self):
settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
crawler = get_crawler(settings_dict=settings)
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
self.assertIsNone(spider._get_sitemap_body(response))
def test_compression_bomb_spider_attr(self):
class DownloadMaxSizeSpider(self.spider_class):
download_maxsize = 10_000_000
crawler = get_crawler()
spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
self.assertIsNone(spider._get_sitemap_body(response))
def test_compression_bomb_request_meta(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(
url="https://example.com", meta={"download_maxsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
self.assertIsNone(spider._get_sitemap_body(response))
def test_download_warnsize_setting(self):
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
crawler = get_crawler(settings_dict=settings)
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
def test_download_warnsize_spider_attr(self):
class DownloadWarnSizeSpider(self.spider_class):
download_warnsize = 10_000_000
crawler = get_crawler()
spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(
url="https://example.com", meta={"download_warnsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
def test_download_warnsize_request_meta(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(
url="https://example.com", meta={"download_warnsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
class DeprecationTest(unittest.TestCase):
def test_crawl_spider(self):