diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index aa3abe853..0e5e215ac 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -29,7 +29,10 @@ logger = getLogger(__name__) ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: pass else: diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 7c40d0a02..84c255c28 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -5,7 +5,10 @@ from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning try: - import brotli + try: + import brotli + except ImportError: + import brotlicffi as brotli except ImportError: pass else: @@ -17,9 +20,9 @@ else: "You have brotlipy installed, and Scrapy will use it, but " "Scrapy support for brotlipy is deprecated and will stop " "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi (not " - "currently supported by Scrapy). Please, uninstall brotlipy " - "and install brotli instead. brotlipy has the same import " + "deprecated, it has been superseded by brotlicffi. " + "Please, uninstall brotlipy " + "and install brotli or brotlicffi instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " "discouraged." ), diff --git a/tests/requirements.txt b/tests/requirements.txt index 5b75674f5..ca5f6ddbd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,8 +11,7 @@ uvloop; platform_system != "Windows" bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 -brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests +brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index ae5569d0a..7c36f748e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -130,7 +130,10 @@ class HttpCompressionTest(TestCase): def test_process_response_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") response = self._getresponse("br") @@ -448,7 +451,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_setting("br") @@ -486,7 +492,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_spider_attr("br") @@ -522,7 +531,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_request_meta("br") @@ -568,7 +580,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_setting("br") @@ -616,7 +631,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_spider_attr("br") @@ -662,7 +680,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_request_meta("br")