From cab1016bb6f719b15043f65502c63fbaa191df36 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 20:24:17 -0300 Subject: [PATCH 1/8] Add brotlicffi support --- scrapy/utils/_compression.py | 2 ++ ...st_downloadermiddleware_httpcompression.py | 35 +++++++++++++++---- tox.ini | 1 + 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 7c40d0a02..7896f4c01 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -6,6 +6,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning try: import brotli +except ImportError: + import brotlicffi as brotli except ImportError: pass else: diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index ae5569d0a..7c36f748e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -130,7 +130,10 @@ class HttpCompressionTest(TestCase): def test_process_response_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") response = self._getresponse("br") @@ -448,7 +451,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_setting("br") @@ -486,7 +492,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_spider_attr("br") @@ -522,7 +531,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_request_meta("br") @@ -568,7 +580,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_setting("br") @@ -616,7 +631,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_spider_attr("br") @@ -662,7 +680,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_request_meta("br") diff --git a/tox.ini b/tox.ini index 237aa489c..6b804b78c 100644 --- a/tox.ini +++ b/tox.ini @@ -162,6 +162,7 @@ basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} PyPyDispatcher==2.1.0 + brotlicffi commands = pytest --durations=10 scrapy tests install_command = {[pinned]install_command} From 3421823dce94a693ee86915110d899d8da6f3e9f Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 20:26:23 -0300 Subject: [PATCH 2/8] Nested try-except block --- scrapy/utils/_compression.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 7896f4c01..477573588 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -5,9 +5,10 @@ from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning try: - import brotli -except ImportError: - import brotlicffi as brotli + try: + import brotli + except ImportError: + import brotlicffi as brotli except ImportError: pass else: From a52429ae08ec15d70f7f3e2079d32933bb639d6b Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:12:25 -0300 Subject: [PATCH 3/8] Update disclaimer --- scrapy/utils/_compression.py | 4 ++-- tox.ini | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 477573588..4b3fd342d 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -20,8 +20,8 @@ else: "You have brotlipy installed, and Scrapy will use it, but " "Scrapy support for brotlipy is deprecated and will stop " "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi (not " - "currently supported by Scrapy). Please, uninstall brotlipy " + "deprecated, it has been superseded by brotlicffi " + "Please, uninstall brotlipy " "and install brotli instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " "discouraged." diff --git a/tox.ini b/tox.ini index 6b804b78c..9cf3c92ad 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - brotli + brotli; implementation_name != 'pypy' zstandard [testenv:extra-deps-pinned] From 16864ea602ebc3d1a764aaf6c101a5f20ff57bee Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:18:24 -0300 Subject: [PATCH 4/8] Remove PyPy condition --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9cf3c92ad..6b804b78c 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - brotli; implementation_name != 'pypy' + brotli zstandard [testenv:extra-deps-pinned] From 532cd2eabd8b280e64a1087c49b8f5eb5f05530f Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:30:20 -0300 Subject: [PATCH 5/8] Use brotlicffi for PyPy --- tests/requirements.txt | 3 +-- tox.ini | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 5b75674f5..ca5f6ddbd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,8 +11,7 @@ uvloop; platform_system != "Windows" bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 -brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests +brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index 6b804b78c..237aa489c 100644 --- a/tox.ini +++ b/tox.ini @@ -162,7 +162,6 @@ basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} PyPyDispatcher==2.1.0 - brotlicffi commands = pytest --durations=10 scrapy tests install_command = {[pinned]install_command} From 7f1fbdba3cc6f118cbf11285ed26e488f854aed1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 22:11:11 -0300 Subject: [PATCH 6/8] Check brotlicffi for ACCEPTED_ENCODINGS --- scrapy/downloadermiddlewares/httpcompression.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index aa3abe853..0e5e215ac 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -29,7 +29,10 @@ logger = getLogger(__name__) ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: pass else: From 7be919138d84ec00feb80a78e13721dff998c10c Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 6 Mar 2024 05:49:31 -0300 Subject: [PATCH 7/8] Update scrapy/utils/_compression.py Co-authored-by: Andrey Rakhmatullin --- scrapy/utils/_compression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 4b3fd342d..349fd9ac0 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -20,7 +20,7 @@ else: "You have brotlipy installed, and Scrapy will use it, but " "Scrapy support for brotlipy is deprecated and will stop " "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi " + "deprecated, it has been superseded by brotlicffi. " "Please, uninstall brotlipy " "and install brotli instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " From 99f7165c63a8a2dba72090f65ba1093d476d669a Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:02:01 -0300 Subject: [PATCH 8/8] Update scrapy/utils/_compression.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/_compression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 349fd9ac0..84c255c28 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -22,7 +22,7 @@ else: "working in a future version of Scrapy. brotlipy itself is " "deprecated, it has been superseded by brotlicffi. " "Please, uninstall brotlipy " - "and install brotli instead. brotlipy has the same import " + "and install brotli or brotlicffi instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " "discouraged." ),