From 609f64c55d8603710218113ad42b1bbf940b3978 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 19:24:14 +0200 Subject: [PATCH] Make brotli a hard dependency (#7929) --- .github/workflows/tests-ubuntu.yml | 1 + docs/intro/install.rst | 2 - docs/topics/downloader-middleware.rst | 9 ++- pyproject.toml | 6 +- .../downloadermiddlewares/httpcompression.py | 24 +------- scrapy/utils/_compression.py | 9 ++- tests/test_command_shell.py | 6 +- ...st_downloadermiddleware_httpcompression.py | 55 ------------------- tox.ini | 8 +-- 9 files changed, 18 insertions(+), 102 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index cd726a2fe..b58b2af7b 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -86,6 +86,7 @@ jobs: - python-version: pypy3.11-7.3.20 env: TOXENV: pypy3-extra-deps + coverage: true - python-version: "3.14" env: TOXENV: botocore diff --git a/docs/intro/install.rst b/docs/intro/install.rst index cba8c15a1..866c5abe2 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -111,8 +111,6 @@ The following extras are available: - Provides * - ``bpython`` - :ref:`bpython shell ` - * - ``brotli`` - - :ref:`Brotli response decompression ` * - ``gcs`` - :ref:`Google Cloud Storage ` for :ref:`feed exports ` and diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fcfe7fd29..66a0164d9 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -741,14 +741,13 @@ HttpCompressionMiddleware .. class:: HttpCompressionMiddleware - This middleware allows compressed (gzip, deflate) traffic to be + This middleware allows compressed (gzip, deflate, `brotli`_) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ responses with - the :ref:`brotli ` extra, and `zstd-compressed`_ - responses with the :ref:`zstd ` extra. + This middleware also supports decoding `zstd-compressed`_ responses with + the :ref:`zstd ` extra. -.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt +.. _brotli: https://www.ietf.org/rfc/rfc7932.txt .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt diff --git a/pyproject.toml b/pyproject.toml index 0dcbade90..4edfb7ec8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,8 @@ dependencies = [ # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', + 'brotli>=1.2.0; implementation_name != "pypy"', + 'brotlicffi>=1.2.0.0; implementation_name == "pypy"', ] classifiers = [ "Development Status :: 5 - Production/Stable", @@ -62,10 +64,6 @@ Tracker = "https://github.com/scrapy/scrapy/issues" [project.optional-dependencies] bpython = ["bpython>=0.7.1"] -brotli = [ - "brotli>=1.2.0; implementation_name != 'pypy'", - "brotlicffi>=1.2.0.0; implementation_name == 'pypy'", -] gcs = ["google-cloud-storage>=1.29.0"] httpx = ["httpx2[http2,socks]>=2.0.0"] images = ["Pillow>=8.3.2"] diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b99c323c5..0045ddcaa 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -30,27 +30,7 @@ if TYPE_CHECKING: logger = getLogger(__name__) -ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"] - -try: - try: - import brotli - except ImportError: - import brotlicffi as brotli -except ImportError: - pass -else: - try: - brotli.Decompressor.can_accept_more_data # noqa: B018 - except AttributeError: # pragma: no cover - warnings.warn( - "You have brotli installed. But 'br' encoding support now requires " - "brotli's or brotlicffi's version >= 1.2.0. Please upgrade " - "brotli/brotlicffi to make Scrapy decode 'br' encoded responses.", - stacklevel=2, - ) - else: - ACCEPTED_ENCODINGS.append(b"br") +ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate", b"br"] if find_spec("zstandard") is not None: ACCEPTED_ENCODINGS.append(b"zstd") @@ -205,8 +185,6 @@ class HttpCompressionMiddleware: f"{self.__class__.__name__} cannot decode the response for {response.url} " f"from unsupported encoding(s) '{encodings_str}'." ) - if b"br" in encodings: - msg += " You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'." if b"zstd" in encodings: msg += " You need to install zstandard to decode 'zstd'." logger.warning(msg) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 4767c29f2..ac98a61c8 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -2,11 +2,10 @@ import contextlib import zlib from io import BytesIO -with contextlib.suppress(ImportError): - try: - import brotli - except ImportError: - import brotlicffi as brotli +try: + import brotli +except ImportError: + import brotlicffi as brotli with contextlib.suppress(ImportError): import zstandard diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 29667a1ae..6bc1ebbbb 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -201,7 +201,7 @@ class TestInteractiveShell: env = os.environ.copy() env["SCRAPY_PYTHON_SHELL"] = "python" logfile = BytesIO() - p = PopenSpawn(args, env=env, timeout=5) + p = PopenSpawn(args, env=env, timeout=60) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") p.sendline(f"fetch('{mockserver.url('/')}')") @@ -235,7 +235,7 @@ class TestInteractiveShell: def _run_interactive_shell(self, env: dict[str, str]) -> str: args = (sys.executable, "-m", "scrapy.cmdline", "shell") logfile = BytesIO() - p = PopenSpawn(args, env=env, timeout=5) + p = PopenSpawn(args, env=env, timeout=60) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") p.sendeof() @@ -256,7 +256,7 @@ class TestInteractiveShell: self._isolate_config(env, config_home) args = (sys.executable, "-m", "scrapy.cmdline", "shell") logfile = BytesIO() - p = PopenSpawn(args, env=env, timeout=10) + p = PopenSpawn(args, env=env, timeout=60) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") # The standard Python shell never imports IPython, whereas the IPython diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index fa0707491..a43bb51ba 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -52,20 +52,6 @@ FORMAT = { } -def _skip_if_no_br() -> None: - try: - try: - import brotli # noqa: PLC0415 - - brotli.Decompressor.can_accept_more_data - except (ImportError, AttributeError): - import brotlicffi # noqa: PLC0415 - - brotlicffi.Decompressor.can_accept_more_data - except (ImportError, AttributeError): - pytest.skip("no brotli support") - - def _skip_if_no_zstd() -> None: pytest.importorskip("zstandard") @@ -161,8 +147,6 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_bytes", 74837) def test_process_response_br(self): - _skip_if_no_br() - response = self._getresponse("br") assert response.request request = response.request @@ -174,32 +158,6 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) - def test_process_response_br_unsupported(self, caplog: pytest.LogCaptureFixture): - if find_spec("brotli") is not None or find_spec("brotlicffi") is not None: - pytest.skip("Requires not having brotli support") - response = self._getresponse("br") - assert response.request - request = response.request - assert response.headers["Content-Encoding"] == b"br" - caplog.clear() - with caplog.at_level( - WARNING, logger="scrapy.downloadermiddlewares.httpcompression" - ): - newresponse = self.mw.process_response(request, response) - assert caplog.record_tuples == [ - ( - "scrapy.downloadermiddlewares.httpcompression", - WARNING, - ( - "HttpCompressionMiddleware cannot decode the response for " - "http://scrapytest.org/ from unsupported encoding(s) 'br'. " - "You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'." - ), - ), - ] - assert newresponse is not response - assert newresponse.headers.getlist("Content-Encoding") == [b"br"] - def test_process_response_zstd(self): _skip_if_no_zstd() @@ -550,8 +508,6 @@ class TestHttpCompression: assert cause.decompressed_size < 1_100_000 def test_compression_bomb_setting_br(self): - _skip_if_no_br() - self._test_compression_bomb_setting("br") def test_compression_bomb_setting_deflate(self): @@ -609,8 +565,6 @@ class TestHttpCompression: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr_br(self): - _skip_if_no_br() - self._test_compression_bomb_spider_attr("br") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @@ -643,8 +597,6 @@ class TestHttpCompression: assert cause.decompressed_size < 1_100_000 def test_compression_bomb_request_meta_br(self): - _skip_if_no_br() - self._test_compression_bomb_request_meta("br") def test_compression_bomb_request_meta_deflate(self): @@ -689,8 +641,6 @@ class TestHttpCompression: def test_download_warnsize_setting_br( self, caplog: pytest.LogCaptureFixture ) -> None: - _skip_if_no_br() - self._test_download_warnsize_setting(caplog, "br") def test_download_warnsize_setting_deflate( @@ -744,8 +694,6 @@ class TestHttpCompression: def test_download_warnsize_spider_attr_br( self, caplog: pytest.LogCaptureFixture ) -> None: - _skip_if_no_br() - self._test_download_warnsize_spider_attr(caplog, "br") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @@ -799,8 +747,6 @@ class TestHttpCompression: def test_download_warnsize_request_meta_br( self, caplog: pytest.LogCaptureFixture ) -> None: - _skip_if_no_br() - self._test_download_warnsize_request_meta(caplog, "br") def test_download_warnsize_request_meta_deflate( @@ -834,7 +780,6 @@ class TestHttpCompression: return new_response def test_process_truncated_response_br(self): - _skip_if_no_br() resp = self._get_truncated_response("br") assert resp.body.startswith(b"= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests - brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests google-cloud-storage httpx2[http2,socks] ipython @@ -189,8 +189,6 @@ deps = Twisted[http2]==21.7.0 boto3==1.20.0 bpython==0.7.1 - brotli==1.2.0; implementation_name != "pypy" - brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 httpx2[http2,socks]==2.0.0 ipython==8.15.0 @@ -291,7 +289,6 @@ commands = basepython = pypy3 deps = {[testenv:extra-deps]deps} -commands = {[testenv:pypy3]commands} [testenv:min-pypy3] basepython = pypy3.11 @@ -301,6 +298,7 @@ deps = pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 + brotlicffi==1.2.0.0 cryptography==44.0.2 cssselect==0.9.1 itemadapter==0.1.0