RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make brotli a hard dependency (#7929)

This commit is contained in:
Adrian 2026-08-09 19:24:14 +02:00 committed by GitHub
parent 5427080f48
commit 609f64c55d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 18 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/tests-ubuntu.yml vendored
View File

@ -86,6 +86,7 @@ jobs:
- python-version: pypy3.11-7.3.20
env:
TOXENV: pypy3-extra-deps
coverage: true
- python-version: "3.14"
env:
TOXENV: botocore

2
docs/intro/install.rst
View File

@ -111,8 +111,6 @@ The following extras are available:
- Provides
* - ``bpython``
- :ref:`bpython shell <shell-config>`
* - ``brotli``
- :ref:`Brotli response decompression <http-compression>`
* - ``gcs``
- :ref:`Google Cloud Storage <topics-feed-storage-gcs>` for
:ref:`feed exports <topics-feed-exports>` and

9
docs/topics/downloader-middleware.rst
View File

@ -741,14 +741,13 @@ HttpCompressionMiddleware
.. class:: HttpCompressionMiddleware
This middleware allows compressed (gzip, deflate) traffic to be
This middleware allows compressed (gzip, deflate, `brotli`_) traffic to be
sent/received from web sites.
This middleware also supports decoding `brotli-compressed`_ responses with
the :ref:`brotli <extras>` extra, and `zstd-compressed`_
responses with the :ref:`zstd <extras>` extra.
This middleware also supports decoding `zstd-compressed`_ responses with
the :ref:`zstd <extras>` extra.
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
.. _brotli: https://www.ietf.org/rfc/rfc7932.txt
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt

6
pyproject.toml
View File

@ -26,6 +26,8 @@ dependencies = [
# Platform-specific dependencies
'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',
'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"',
'brotli>=1.2.0; implementation_name != "pypy"',
'brotlicffi>=1.2.0.0; implementation_name == "pypy"',
]
classifiers = [
"Development Status :: 5 - Production/Stable",
@ -62,10 +64,6 @@ Tracker = "https://github.com/scrapy/scrapy/issues"
[project.optional-dependencies]
bpython = ["bpython>=0.7.1"]
brotli = [
"brotli>=1.2.0; implementation_name != 'pypy'",
"brotlicffi>=1.2.0.0; implementation_name == 'pypy'",
]
gcs = ["google-cloud-storage>=1.29.0"]
httpx = ["httpx2[http2,socks]>=2.0.0"]
images = ["Pillow>=8.3.2"]

24
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -30,27 +30,7 @@ if TYPE_CHECKING:
logger = getLogger(__name__)
ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"]
try:
try:
import brotli
except ImportError:
import brotlicffi as brotli
except ImportError:
pass
else:
try:
brotli.Decompressor.can_accept_more_data # noqa: B018
except AttributeError: # pragma: no cover
warnings.warn(
"You have brotli installed. But 'br' encoding support now requires "
"brotli's or brotlicffi's version >= 1.2.0. Please upgrade "
"brotli/brotlicffi to make Scrapy decode 'br' encoded responses.",
stacklevel=2,
)
else:
ACCEPTED_ENCODINGS.append(b"br")
ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate", b"br"]
if find_spec("zstandard") is not None:
ACCEPTED_ENCODINGS.append(b"zstd")
@ -205,8 +185,6 @@ class HttpCompressionMiddleware:
f"{self.__class__.__name__} cannot decode the response for {response.url} "
f"from unsupported encoding(s) '{encodings_str}'."
)
if b"br" in encodings:
msg += " You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'."
if b"zstd" in encodings:
msg += " You need to install zstandard to decode 'zstd'."
logger.warning(msg)

9
scrapy/utils/_compression.py
View File

@ -2,11 +2,10 @@ import contextlib
import zlib
from io import BytesIO
with contextlib.suppress(ImportError):
try:
import brotli
except ImportError:
import brotlicffi as brotli
try:
import brotli
except ImportError:
import brotlicffi as brotli
with contextlib.suppress(ImportError):
import zstandard

6
tests/test_command_shell.py
View File

@ -201,7 +201,7 @@ class TestInteractiveShell:
env = os.environ.copy()
env["SCRAPY_PYTHON_SHELL"] = "python"
logfile = BytesIO()
p = PopenSpawn(args, env=env, timeout=5)
p = PopenSpawn(args, env=env, timeout=60)
p.logfile_read = logfile
p.expect_exact("Available Scrapy objects")
p.sendline(f"fetch('{mockserver.url('/')}')")
@ -235,7 +235,7 @@ class TestInteractiveShell:
def _run_interactive_shell(self, env: dict[str, str]) -> str:
args = (sys.executable, "-m", "scrapy.cmdline", "shell")
logfile = BytesIO()
p = PopenSpawn(args, env=env, timeout=5)
p = PopenSpawn(args, env=env, timeout=60)
p.logfile_read = logfile
p.expect_exact("Available Scrapy objects")
p.sendeof()
@ -256,7 +256,7 @@ class TestInteractiveShell:
self._isolate_config(env, config_home)
args = (sys.executable, "-m", "scrapy.cmdline", "shell")
logfile = BytesIO()
p = PopenSpawn(args, env=env, timeout=10)
p = PopenSpawn(args, env=env, timeout=60)
p.logfile_read = logfile
p.expect_exact("Available Scrapy objects")
# The standard Python shell never imports IPython, whereas the IPython

55
tests/test_downloadermiddleware_httpcompression.py
View File

@ -52,20 +52,6 @@ FORMAT = {
}
def _skip_if_no_br() -> None:
try:
try:
import brotli # noqa: PLC0415
brotli.Decompressor.can_accept_more_data
except (ImportError, AttributeError):
import brotlicffi # noqa: PLC0415
brotlicffi.Decompressor.can_accept_more_data
except (ImportError, AttributeError):
pytest.skip("no brotli support")
def _skip_if_no_zstd() -> None:
pytest.importorskip("zstandard")
@ -161,8 +147,6 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_bytes", 74837)
def test_process_response_br(self):
_skip_if_no_br()
response = self._getresponse("br")
assert response.request
request = response.request
@ -174,32 +158,6 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
def test_process_response_br_unsupported(self, caplog: pytest.LogCaptureFixture):
if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:
pytest.skip("Requires not having brotli support")
response = self._getresponse("br")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"br"
caplog.clear()
with caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
):
newresponse = self.mw.process_response(request, response)
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
WARNING,
(
"HttpCompressionMiddleware cannot decode the response for "
"http://scrapytest.org/ from unsupported encoding(s) 'br'. "
"You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'."
),
),
]
assert newresponse is not response
assert newresponse.headers.getlist("Content-Encoding") == [b"br"]
def test_process_response_zstd(self):
_skip_if_no_zstd()
@ -550,8 +508,6 @@ class TestHttpCompression:
assert cause.decompressed_size < 1_100_000
def test_compression_bomb_setting_br(self):
_skip_if_no_br()
self._test_compression_bomb_setting("br")
def test_compression_bomb_setting_deflate(self):
@ -609,8 +565,6 @@ class TestHttpCompression:
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr_br(self):
_skip_if_no_br()
self._test_compression_bomb_spider_attr("br")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@ -643,8 +597,6 @@ class TestHttpCompression:
assert cause.decompressed_size < 1_100_000
def test_compression_bomb_request_meta_br(self):
_skip_if_no_br()
self._test_compression_bomb_request_meta("br")
def test_compression_bomb_request_meta_deflate(self):
@ -689,8 +641,6 @@ class TestHttpCompression:
def test_download_warnsize_setting_br(
self, caplog: pytest.LogCaptureFixture
) -> None:
_skip_if_no_br()
self._test_download_warnsize_setting(caplog, "br")
def test_download_warnsize_setting_deflate(
@ -744,8 +694,6 @@ class TestHttpCompression:
def test_download_warnsize_spider_attr_br(
self, caplog: pytest.LogCaptureFixture
) -> None:
_skip_if_no_br()
self._test_download_warnsize_spider_attr(caplog, "br")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@ -799,8 +747,6 @@ class TestHttpCompression:
def test_download_warnsize_request_meta_br(
self, caplog: pytest.LogCaptureFixture
) -> None:
_skip_if_no_br()
self._test_download_warnsize_request_meta(caplog, "br")
def test_download_warnsize_request_meta_deflate(
@ -834,7 +780,6 @@ class TestHttpCompression:
return new_response
def test_process_truncated_response_br(self):
_skip_if_no_br()
resp = self._get_truncated_response("br")
assert resp.body.startswith(b"<!DOCTYPE")

8
tox.ini
View File

@ -137,6 +137,8 @@ deps =
pytest==8.4.0
Protego==0.1.15
Twisted==21.7.0
brotli==1.2.0; implementation_name != "pypy"
brotlicffi==1.2.0.0; implementation_name == "pypy"
cryptography==37.0.0
cssselect==0.9.1
httpx2==2.0.0
@ -171,8 +173,6 @@ deps =
Twisted[http2]
boto3
bpython # optional for shell wrapper tests
brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests
brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests
google-cloud-storage
httpx2[http2,socks]
ipython
@ -189,8 +189,6 @@ deps =
Twisted[http2]==21.7.0
boto3==1.20.0
bpython==0.7.1
brotli==1.2.0; implementation_name != "pypy"
brotlicffi==1.2.0.0; implementation_name == "pypy"
google-cloud-storage==1.29.0
httpx2[http2,socks]==2.0.0
ipython==8.15.0
@ -291,7 +289,6 @@ commands =
basepython = pypy3
deps =
{[testenv:extra-deps]deps}
commands = {[testenv:pypy3]commands}
[testenv:min-pypy3]
basepython = pypy3.11
@ -301,6 +298,7 @@ deps =
pytest==8.4.0
Protego==0.1.15
Twisted==21.7.0
brotlicffi==1.2.0.0
cryptography==44.0.2
cssselect==0.9.1
itemadapter==0.1.0