Merge pull request #6269 from Laerte/master

Add brotlicffi support

scrapy/downloadermiddlewares/httpcompression.py

@@ -29,7 +29,10 @@ logger = getLogger(__name__)
 ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
 
 try:
-    import brotli  # noqa: F401
+    try:
+        import brotli  # noqa: F401
+    except ImportError:
+        import brotlicffi  # noqa: F401
 except ImportError:
     pass
 else:

scrapy/utils/_compression.py

@@ -5,7 +5,10 @@ from warnings import warn
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 try:
-    import brotli
+    try:
+        import brotli
+    except ImportError:
+        import brotlicffi as brotli
 except ImportError:
     pass
 else:
@@ -17,9 +20,9 @@ else:
                 "You have brotlipy installed, and Scrapy will use it, but "
                 "Scrapy support for brotlipy is deprecated and will stop "
                 "working in a future version of Scrapy. brotlipy itself is "
-                "deprecated, it has been superseded by brotlicffi (not "
-                "currently supported by Scrapy). Please, uninstall brotlipy "
-                "and install brotli instead. brotlipy has the same import "
+                "deprecated, it has been superseded by brotlicffi. "
+                "Please, uninstall brotlipy "
+                "and install brotli or brotlicffi instead. brotlipy has the same import "
                 "name as brotli, so keeping both installed is strongly "
                 "discouraged."
             ),

tests/requirements.txt

@@ -11,8 +11,7 @@ uvloop; platform_system != "Windows"
 
 bpython  # optional for shell wrapper tests
 brotli; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
-# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072
-brotli==1.0.9; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests
+brotlicffi; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests
 zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"

tests/test_downloadermiddleware_httpcompression.py

@@ -130,7 +130,10 @@ class HttpCompressionTest(TestCase):
 
     def test_process_response_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         response = self._getresponse("br")
@@ -448,7 +451,10 @@ class HttpCompressionTest(TestCase):
 
     def test_compression_bomb_setting_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_compression_bomb_setting("br")
@@ -486,7 +492,10 @@ class HttpCompressionTest(TestCase):
 
     def test_compression_bomb_spider_attr_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_compression_bomb_spider_attr("br")
@@ -522,7 +531,10 @@ class HttpCompressionTest(TestCase):
 
     def test_compression_bomb_request_meta_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_compression_bomb_request_meta("br")
@@ -568,7 +580,10 @@ class HttpCompressionTest(TestCase):
 
     def test_download_warnsize_setting_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_download_warnsize_setting("br")
@@ -616,7 +631,10 @@ class HttpCompressionTest(TestCase):
 
     def test_download_warnsize_spider_attr_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_download_warnsize_spider_attr("br")
@@ -662,7 +680,10 @@ class HttpCompressionTest(TestCase):
 
     def test_download_warnsize_request_meta_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_download_warnsize_request_meta("br")