diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 5610595d3..14531df3f 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,10 +1,40 @@ import zlib from io import BytesIO +from warnings import warn + +from scrapy.exceptions import ScrapyDeprecationWarning try: import brotli except ImportError: pass +else: + try: + brotli.Decompressor.process + except AttributeError: + + warn( + ( + "You have brotlipy installed, and Scrapy will use it, but " + "Scrapy support for brotlipy is deprecated and will stop " + "working in a future version of Scrapy. brotlipy itself is " + "deprecated, it has been superseded by brotlicffi (not " + "currently supported by Scrapy). Please, uninstall brotlipy " + "and install brotli instead. brotlipy has the same import " + "name as brotli, so keeping both installed is strongly " + "discouraged." + ), + ScrapyDeprecationWarning, + ) + + def _brotli_decompress(decompressor, data): + return decompressor.decompress(data) + + else: + + def _brotli_decompress(decompressor, data): + return decompressor.process(data) + try: import zstandard @@ -61,7 +91,7 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: decompressed_size = 0 while output_chunk: input_chunk = input_stream.read(_CHUNK_SIZE) - output_chunk = decompressor.process(input_chunk) + output_chunk = _brotli_decompress(decompressor, input_chunk) decompressed_size += len(output_chunk) if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( diff --git a/tox.ini b/tox.ini index 4ed9b3bd7..237aa489c 100644 --- a/tox.ini +++ b/tox.ini @@ -135,6 +135,7 @@ deps = google-cloud-storage==1.29.0 Pillow==7.1.0 robotexclusionrulesparser==1.6.2 + brotlipy install_command = {[pinned]install_command} setenv = {[pinned]setenv}