Replace import tests with find_spec() or importorskip(). (#7755)

conftest.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-import importlib
+from importlib.util import find_spec
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -50,9 +50,7 @@ if not H2_ENABLED:
         )
     )
 
-try:
-    import httpx  # noqa: F401
-except ImportError:
+if not find_spec("httpx"):
     collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
 
 
@@ -130,11 +128,8 @@ def pytest_runtest_setup(item):
     ]
 
     for module in optional_deps:
-        if item.get_closest_marker(f"requires_{module}"):
-            try:
-                importlib.import_module(module)
-            except ImportError:
-                pytest.skip(f"{module} is not installed")
+        if item.get_closest_marker(f"requires_{module}") and find_spec(module) is None:
+            pytest.skip(f"{module} is not installed")
 
     if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:
         pytest.skip("mitmdump is not available")

scrapy/downloadermiddlewares/httpcompression.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import warnings
+from importlib.util import find_spec
 from itertools import chain
 from logging import getLogger
 from typing import TYPE_CHECKING, Any
@@ -51,11 +52,7 @@ else:
     else:
         ACCEPTED_ENCODINGS.append(b"br")
 
-try:
-    import zstandard  # noqa: F401
-except ImportError:
-    pass
-else:
+if find_spec("zstandard") is not None:
     ACCEPTED_ENCODINGS.append(b"zstd")
 
 

scrapy/utils/boto.py

@@ -1,10 +1,7 @@
 """Boto/botocore helpers"""
 
+from importlib.util import find_spec
+
 
 def is_botocore_available() -> bool:
-    try:
-        import botocore  # noqa: F401,PLC0415
-
-        return True
-    except ImportError:
-        return False
+    return find_spec("botocore") is not None

tests/test_downloadermiddleware_httpcompression.py

@@ -1,4 +1,5 @@
 from gzip import GzipFile
+from importlib.util import find_spec
 from io import BytesIO
 from logging import WARNING
 from pathlib import Path
@@ -65,10 +66,7 @@ def _skip_if_no_br() -> None:
 
 
 def _skip_if_no_zstd() -> None:
-    try:
-        import zstandard  # noqa: F401,PLC0415
-    except ImportError:
-        pytest.skip("no zstd support (zstandard)")
+    pytest.importorskip("zstandard")
 
 
 class TestHttpCompression:
@@ -172,17 +170,8 @@ class TestHttpCompression:
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
     def test_process_response_br_unsupported(self):
-        try:
-            try:
-                import brotli  # noqa: F401,PLC0415
-
-                pytest.skip("Requires not having brotli support")
-            except ImportError:
-                import brotlicffi  # noqa: F401,PLC0415
-
-                pytest.skip("Requires not having brotli support")
-        except ImportError:
-            pass
+        if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:
+            pytest.skip("Requires not having brotli support")
         response = self._getresponse("br")
         request = response.request
         assert response.headers["Content-Encoding"] == b"br"
@@ -226,12 +215,8 @@ class TestHttpCompression:
             assert "Content-Encoding" not in newresponse.headers
 
     def test_process_response_zstd_unsupported(self):
-        try:
-            import zstandard  # noqa: F401,PLC0415
-
+        if find_spec("zstandard") is not None:
             pytest.skip("Requires not having zstandard support")
-        except ImportError:
-            pass
         response = self._getresponse("zstd-static-content-size")
         request = response.request
         assert response.headers["Content-Encoding"] == b"zstd"

tests/test_feedexport_storages.py

@@ -436,10 +436,7 @@ class TestS3FeedStorage:
 
 class TestGCSFeedStorage:
     def test_parse_settings(self):
-        try:
-            from google.cloud.storage import Client  # noqa: F401,PLC0415
-        except ImportError:
-            pytest.skip("GCSFeedStorage requires google-cloud-storage")
+        pytest.importorskip("google.cloud.storage")
 
         settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"}
         crawler = get_crawler(settings_dict=settings)
@@ -450,10 +447,7 @@ class TestGCSFeedStorage:
         assert storage.blob_name == "export.csv"
 
     def test_parse_empty_acl(self):
-        try:
-            from google.cloud.storage import Client  # noqa: F401,PLC0415
-        except ImportError:
-            pytest.skip("GCSFeedStorage requires google-cloud-storage")
+        pytest.importorskip("google.cloud.storage")
 
         settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}
         crawler = get_crawler(settings_dict=settings)
@@ -467,10 +461,7 @@ class TestGCSFeedStorage:
 
     @coroutine_test
     async def test_store(self):
-        try:
-            from google.cloud.storage import Client  # noqa: F401,PLC0415
-        except ImportError:
-            pytest.skip("GCSFeedStorage requires google-cloud-storage")
+        pytest.importorskip("google.cloud.storage")
 
         uri = "gs://mybucket/export.csv"
         project_id = "myproject-123"
@@ -492,10 +483,7 @@ class TestGCSFeedStorage:
 
     @coroutine_test
     async def test_store_closes_file_on_upload_error(self):
-        try:
-            from google.cloud.storage import Client  # noqa: F401,PLC0415
-        except ImportError:
-            pytest.skip("GCSFeedStorage requires google-cloud-storage")
+        pytest.importorskip("google.cloud.storage")
 
         uri = "gs://mybucket/export.csv"
         project_id = "myproject-123"

tests/utils/robotstxt.py

@@ -1,12 +1,7 @@
 from __future__ import annotations
 
+from importlib.util import find_spec
+
 
 def rerp_available() -> bool:
-    # check if robotexclusionrulesparser is installed
-    try:
-        from robotexclusionrulesparser import (  # noqa: PLC0415
-            RobotExclusionRulesParser,  # noqa: F401
-        )
-    except ImportError:
-        return False
-    return True
+    return find_spec("robotexclusionrulesparser") is not None