From cab1016bb6f719b15043f65502c63fbaa191df36 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 20:24:17 -0300 Subject: [PATCH] Add brotlicffi support --- scrapy/utils/_compression.py | 2 ++ ...st_downloadermiddleware_httpcompression.py | 35 +++++++++++++++---- tox.ini | 1 + 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 7c40d0a02..7896f4c01 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -6,6 +6,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning try: import brotli +except ImportError: + import brotlicffi as brotli except ImportError: pass else: diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index ae5569d0a..7c36f748e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -130,7 +130,10 @@ class HttpCompressionTest(TestCase): def test_process_response_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") response = self._getresponse("br") @@ -448,7 +451,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_setting("br") @@ -486,7 +492,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_spider_attr("br") @@ -522,7 +531,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_request_meta("br") @@ -568,7 +580,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_setting("br") @@ -616,7 +631,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_spider_attr("br") @@ -662,7 +680,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_request_meta("br") diff --git a/tox.ini b/tox.ini index 237aa489c..6b804b78c 100644 --- a/tox.ini +++ b/tox.ini @@ -162,6 +162,7 @@ basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} PyPyDispatcher==2.1.0 + brotlicffi commands = pytest --durations=10 scrapy tests install_command = {[pinned]install_command}