diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 06da46ca1..5d13f0c1f 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -68,6 +68,9 @@ jobs: - python-version: "3.13" env: TOXENV: botocore + - python-version: "3.13" + env: + TOXENV: mitmproxy steps: - uses: actions/checkout@v4 diff --git a/conftest.py b/conftest.py index 8e0c429a0..9ffcff726 100644 --- a/conftest.py +++ b/conftest.py @@ -116,6 +116,16 @@ def requires_boto3(request): pytest.skip("boto3 is not installed") +@pytest.fixture(autouse=True) +def requires_mitmproxy(request): + if not request.node.get_closest_marker("requires_mitmproxy"): + return + try: + import mitmproxy # noqa: F401 + except ImportError: + pytest.skip("mitmproxy is not installed") + + def pytest_configure(config): if config.getoption("--reactor") != "default": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/pyproject.toml b/pyproject.toml index e1cfec355..f81fb295e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -243,6 +243,7 @@ markers = [ "requires_uvloop: marks tests as only enabled when uvloop is known to be working", "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", + "requires_mitmproxy: marks tests that need mitmproxy", ] filterwarnings = [ "ignore::DeprecationWarning:twisted.web.static" diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 58891b952..4190a6547 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from itertools import chain from logging import getLogger from typing import TYPE_CHECKING, Any @@ -29,14 +30,20 @@ logger = getLogger(__name__) ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"] try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 + import brotli except ImportError: pass else: - ACCEPTED_ENCODINGS.append(b"br") + try: + brotli.Decompressor.can_accept_more_data + except AttributeError: # pragma: no cover + warnings.warn( + "You have brotli installed. But 'br' encoding support now requires " + "brotli version >= 1.2.0. Please upgrade brotli version to make Scrapy " + "decode 'br' encoded responses.", + ) + else: + ACCEPTED_ENCODINGS.append(b"br") try: import zstandard # noqa: F401 @@ -98,13 +105,13 @@ class HttpCompressionMiddleware: decoded_body, content_encoding = self._handle_encoding( response.body, content_encoding, max_size ) - except _DecompressionMaxSizeExceeded: + except _DecompressionMaxSizeExceeded as e: raise IgnoreRequest( f"Ignored response {response} because its body " - f"({len(response.body)} B compressed) exceeded " - f"DOWNLOAD_MAXSIZE ({max_size} B) during " - f"decompression." - ) + f"({len(response.body)} B compressed, " + f"{e.decompressed_size} B decompressed so far) exceeded " + f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression." + ) from e if len(response.body) < warn_size <= len(decoded_body): logger.warning( f"{response} body size after decompression " @@ -187,7 +194,7 @@ class HttpCompressionMiddleware: f"from unsupported encoding(s) '{encodings_str}'." ) if b"br" in encodings: - msg += " You need to install brotli or brotlicffi to decode 'br'." + msg += " You need to install brotli >= 1.2.0 to decode 'br'." if b"zstd" in encodings: msg += " You need to install zstandard to decode 'zstd'." logger.warning(msg) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 6b09f36ff..0a8191cc6 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,42 +1,9 @@ import contextlib import zlib from io import BytesIO -from warnings import warn - -from scrapy.exceptions import ScrapyDeprecationWarning - -try: - try: - import brotli - except ImportError: - import brotlicffi as brotli -except ImportError: - pass -else: - try: - brotli.Decompressor.process - except AttributeError: - warn( - ( - "You have brotlipy installed, and Scrapy will use it, but " - "Scrapy support for brotlipy is deprecated and will stop " - "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi. Please, " - "uninstall brotlipy and install brotli or brotlicffi instead. " - "brotlipy has the same import name as brotli, so keeping both " - "installed is strongly discouraged." - ), - ScrapyDeprecationWarning, - ) - - def _brotli_decompress(decompressor, data): - return decompressor.decompress(data) - - else: - - def _brotli_decompress(decompressor, data): - return decompressor.process(data) +with contextlib.suppress(ImportError): + import brotli with contextlib.suppress(ImportError): import zstandard @@ -46,62 +13,64 @@ _CHUNK_SIZE = 65536 # 64 KiB class _DecompressionMaxSizeExceeded(ValueError): - pass + def __init__(self, decompressed_size: int, max_size: int) -> None: + self.decompressed_size = decompressed_size + self.max_size = max_size + + def __str__(self) -> str: + return ( + f"The number of bytes decompressed so far " + f"({self.decompressed_size} B) exceeded the specified maximum " + f"({self.max_size} B)." + ) + + +def _check_max_size(decompressed_size: int, max_size: int) -> None: + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded(decompressed_size, max_size) def _inflate(data: bytes, *, max_size: int = 0) -> bytes: decompressor = zlib.decompressobj() - raw_decompressor = zlib.decompressobj(wbits=-15) - input_stream = BytesIO(data) + try: + first_chunk = decompressor.decompress(data, max_length=_CHUNK_SIZE) + except zlib.error: + # to work with raw deflate content that may be sent by microsoft servers. + decompressor = zlib.decompressobj(wbits=-15) + first_chunk = decompressor.decompress(data, max_length=_CHUNK_SIZE) + decompressed_size = len(first_chunk) + _check_max_size(decompressed_size, max_size) output_stream = BytesIO() - output_chunk = b"." - decompressed_size = 0 - while output_chunk: - input_chunk = input_stream.read(_CHUNK_SIZE) - try: - output_chunk = decompressor.decompress(input_chunk) - except zlib.error: - if decompressor != raw_decompressor: - # ugly hack to work with raw deflate content that may - # be sent by microsoft servers. For more information, see: - # http://carsten.codimi.de/gzip.yaws/ - # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx - # http://www.gzip.org/zlib/zlib_faq.html#faq38 - decompressor = raw_decompressor - output_chunk = decompressor.decompress(input_chunk) - else: - raise + output_stream.write(first_chunk) + while decompressor.unconsumed_tail: + output_chunk = decompressor.decompress( + decompressor.unconsumed_tail, max_length=_CHUNK_SIZE + ) decompressed_size += len(output_chunk) - if max_size and decompressed_size > max_size: - raise _DecompressionMaxSizeExceeded( - f"The number of bytes decompressed so far " - f"({decompressed_size} B) exceed the specified maximum " - f"({max_size} B)." - ) + _check_max_size(decompressed_size, max_size) output_stream.write(output_chunk) - output_stream.seek(0) - return output_stream.read() + if tail := decompressor.flush(): + decompressed_size += len(tail) + _check_max_size(decompressed_size, max_size) + output_stream.write(tail) + return output_stream.getvalue() def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: decompressor = brotli.Decompressor() - input_stream = BytesIO(data) + first_chunk = decompressor.process(data, output_buffer_limit=_CHUNK_SIZE) + decompressed_size = len(first_chunk) + _check_max_size(decompressed_size, max_size) output_stream = BytesIO() - output_chunk = b"." - decompressed_size = 0 - while output_chunk: - input_chunk = input_stream.read(_CHUNK_SIZE) - output_chunk = _brotli_decompress(decompressor, input_chunk) + output_stream.write(first_chunk) + while not decompressor.is_finished(): + output_chunk = decompressor.process(b"", output_buffer_limit=_CHUNK_SIZE) + if not output_chunk: + break decompressed_size += len(output_chunk) - if max_size and decompressed_size > max_size: - raise _DecompressionMaxSizeExceeded( - f"The number of bytes decompressed so far " - f"({decompressed_size} B) exceed the specified maximum " - f"({max_size} B)." - ) + _check_max_size(decompressed_size, max_size) output_stream.write(output_chunk) - output_stream.seek(0) - return output_stream.read() + return output_stream.getvalue() def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: @@ -113,12 +82,6 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: while output_chunk: output_chunk = stream_reader.read(_CHUNK_SIZE) decompressed_size += len(output_chunk) - if max_size and decompressed_size > max_size: - raise _DecompressionMaxSizeExceeded( - f"The number of bytes decompressed so far " - f"({decompressed_size} B) exceed the specified maximum " - f"({max_size} B)." - ) + _check_max_size(decompressed_size, max_size) output_stream.write(output_chunk) - output_stream.seek(0) - return output_stream.read() + return output_stream.getvalue() diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 85324361c..5f95bccbd 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -5,7 +5,7 @@ from gzip import GzipFile from io import BytesIO from typing import TYPE_CHECKING -from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded +from ._compression import _CHUNK_SIZE, _check_max_size if TYPE_CHECKING: from scrapy.http import Response @@ -31,15 +31,9 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: break raise decompressed_size += len(chunk) - if max_size and decompressed_size > max_size: - raise _DecompressionMaxSizeExceeded( - f"The number of bytes decompressed so far " - f"({decompressed_size} B) exceed the specified maximum " - f"({max_size} B)." - ) + _check_max_size(decompressed_size, max_size) output_stream.write(chunk) - output_stream.seek(0) - return output_stream.read() + return output_stream.getvalue() def gzip_magic_number(response: Response) -> bool: diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index e7427c5ac..7e068587d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -51,6 +51,22 @@ FORMAT = { } +def _skip_if_no_br() -> None: + try: + import brotli + + brotli.Decompressor.can_accept_more_data + except (ImportError, AttributeError): + pytest.skip("no brotli support") + + +def _skip_if_no_zstd() -> None: + try: + import zstandard # noqa: F401 + except ImportError: + pytest.skip("no zstd support (zstandard)") + + class TestHttpCompression: def setup_method(self): self.crawler = get_crawler(Spider) @@ -124,13 +140,7 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_bytes", 74837) def test_process_response_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - raise SkipTest("no brotli") + _skip_if_no_br() response = self._getresponse("br") request = response.request assert response.headers["Content-Encoding"] == b"br" @@ -143,14 +153,9 @@ class TestHttpCompression: def test_process_response_br_unsupported(self): try: - try: - import brotli # noqa: F401 + import brotli # noqa: F401 - raise SkipTest("Requires not having brotli support") - except ImportError: - import brotlicffi # noqa: F401 - - raise SkipTest("Requires not having brotli support") + raise SkipTest("Requires not having brotli support") except ImportError: pass response = self._getresponse("br") @@ -169,7 +174,7 @@ class TestHttpCompression: ( "HttpCompressionMiddleware cannot decode the response for" " http://scrapytest.org/ from unsupported encoding(s) 'br'." - " You need to install brotli or brotlicffi to decode 'br'." + " You need to install brotli >= 1.2.0 to decode 'br'." ), ), ) @@ -177,10 +182,7 @@ class TestHttpCompression: assert newresponse.headers.getlist("Content-Encoding") == [b"br"] def test_process_response_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - raise SkipTest("no zstd support (zstandard)") + _skip_if_no_zstd() raw_content = None for check_key in FORMAT: if not check_key.startswith("zstd-"): @@ -503,24 +505,19 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_bytes", None) def _test_compression_bomb_setting(self, compression_id): - settings = {"DOWNLOAD_MAXSIZE": 10_000_000} + settings = {"DOWNLOAD_MAXSIZE": 1_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") mw = HttpCompressionMiddleware.from_crawler(crawler) mw.open_spider(spider) - response = self._getresponse(f"bomb-{compression_id}") - with pytest.raises(IgnoreRequest): + response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B + with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response, spider) + assert exc_info.value.__cause__.decompressed_size < 1_100_000 def test_compression_bomb_setting_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - raise SkipTest("no brotli") + _skip_if_no_br() self._test_compression_bomb_setting("br") def test_compression_bomb_setting_deflate(self): @@ -530,15 +527,12 @@ class TestHttpCompression: self._test_compression_bomb_setting("gzip") def test_compression_bomb_setting_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - raise SkipTest("no zstd support (zstandard)") + _skip_if_no_zstd() self._test_compression_bomb_setting("zstd") def _test_compression_bomb_spider_attr(self, compression_id): class DownloadMaxSizeSpider(Spider): - download_maxsize = 10_000_000 + download_maxsize = 1_000_000 crawler = get_crawler(DownloadMaxSizeSpider) spider = crawler._create_spider("scrapytest.org") @@ -546,17 +540,12 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - with pytest.raises(IgnoreRequest): + with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response, spider) + assert exc_info.value.__cause__.decompressed_size < 1_100_000 def test_compression_bomb_spider_attr_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - raise SkipTest("no brotli") + _skip_if_no_br() self._test_compression_bomb_spider_attr("br") def test_compression_bomb_spider_attr_deflate(self): @@ -566,10 +555,7 @@ class TestHttpCompression: self._test_compression_bomb_spider_attr("gzip") def test_compression_bomb_spider_attr_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - raise SkipTest("no zstd support (zstandard)") + _skip_if_no_zstd() self._test_compression_bomb_spider_attr("zstd") def _test_compression_bomb_request_meta(self, compression_id): @@ -579,18 +565,13 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - response.meta["download_maxsize"] = 10_000_000 - with pytest.raises(IgnoreRequest): + response.meta["download_maxsize"] = 1_000_000 + with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response, spider) + assert exc_info.value.__cause__.decompressed_size < 1_100_000 def test_compression_bomb_request_meta_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - raise SkipTest("no brotli") + _skip_if_no_br() self._test_compression_bomb_request_meta("br") def test_compression_bomb_request_meta_deflate(self): @@ -600,10 +581,7 @@ class TestHttpCompression: self._test_compression_bomb_request_meta("gzip") def test_compression_bomb_request_meta_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - raise SkipTest("no zstd support (zstandard)") + _skip_if_no_zstd() self._test_compression_bomb_request_meta("zstd") def _test_download_warnsize_setting(self, compression_id): @@ -633,13 +611,7 @@ class TestHttpCompression: ) def test_download_warnsize_setting_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - raise SkipTest("no brotli") + _skip_if_no_br() self._test_download_warnsize_setting("br") def test_download_warnsize_setting_deflate(self): @@ -649,10 +621,7 @@ class TestHttpCompression: self._test_download_warnsize_setting("gzip") def test_download_warnsize_setting_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - raise SkipTest("no zstd support (zstandard)") + _skip_if_no_zstd() self._test_download_warnsize_setting("zstd") def _test_download_warnsize_spider_attr(self, compression_id): @@ -684,13 +653,7 @@ class TestHttpCompression: ) def test_download_warnsize_spider_attr_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - raise SkipTest("no brotli") + _skip_if_no_br() self._test_download_warnsize_spider_attr("br") def test_download_warnsize_spider_attr_deflate(self): @@ -700,10 +663,7 @@ class TestHttpCompression: self._test_download_warnsize_spider_attr("gzip") def test_download_warnsize_spider_attr_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - raise SkipTest("no zstd support (zstandard)") + _skip_if_no_zstd() self._test_download_warnsize_spider_attr("zstd") def _test_download_warnsize_request_meta(self, compression_id): @@ -733,13 +693,7 @@ class TestHttpCompression: ) def test_download_warnsize_request_meta_br(self): - try: - try: - import brotli # noqa: F401 - except ImportError: - import brotlicffi # noqa: F401 - except ImportError: - raise SkipTest("no brotli") + _skip_if_no_br() self._test_download_warnsize_request_meta("br") def test_download_warnsize_request_meta_deflate(self): @@ -749,8 +703,36 @@ class TestHttpCompression: self._test_download_warnsize_request_meta("gzip") def test_download_warnsize_request_meta_zstd(self): - try: - import zstandard # noqa: F401 - except ImportError: - raise SkipTest("no zstd support (zstandard)") + _skip_if_no_zstd() self._test_download_warnsize_request_meta("zstd") + + def _get_truncated_response(self, compression_id): + crawler = get_crawler(Spider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(compression_id) + truncated_body = response.body[: len(response.body) // 2] + response = response.replace(body=truncated_body) + return mw.process_response(response.request, response, spider) + + def test_process_truncated_response_br(self): + _skip_if_no_br() + resp = self._get_truncated_response("br") + assert resp.body.startswith(b"= 8.4.1 # https://github.com/pytest-dev/pytest/pull/13502 - - # mitmproxy does not support PyPy - mitmproxy; implementation_name != "pypy" passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -110,9 +107,6 @@ deps = w3lib==1.17.0 zope.interface==5.1.0 {[test-requirements]deps} - - # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies - # above, hence we do not install it in pinned environments at the moment setenv = _SCRAPY_PINNED=true install_command = @@ -139,8 +133,7 @@ deps = Twisted[http2] boto3 bpython # optional for shell wrapper tests - brotli; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests - brotlicffi; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests + brotli >= 1.2.0 # optional for HTTP compress downloader middleware tests google-cloud-storage ipython robotexclusionrulesparser @@ -154,9 +147,7 @@ deps = Pillow==8.0.0 boto3==1.20.0 bpython==0.7.1 - brotli==0.5.2; implementation_name != "pypy" - brotlicffi==0.8.0; implementation_name == "pypy" - brotlipy + brotli==1.2.0 google-cloud-storage==1.29.0 ipython==2.0.0 robotexclusionrulesparser==1.6.2 @@ -256,7 +247,7 @@ deps = {[testenv]deps} botocore>=1.4.87 commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore [testenv:botocore-pinned] basepython = {[pinned]basepython} @@ -267,4 +258,17 @@ install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy} -m requires_botocore + + +# Run proxy tests that use mitmproxy in a separate env to avoid installing +# numerous mitmproxy deps in other envs (even in extra-deps), as they can +# conflict with other deps we want, or don't want, to have installed there. + +[testenv:mitmproxy] +deps = + {[testenv]deps} + # mitmproxy does not support PyPy + mitmproxy; implementation_name != "pypy" +commands = + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_mitmproxy