diff --git a/conftest.py b/conftest.py index 7403b07b2..1bfb002cf 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,6 @@ from __future__ import annotations -import importlib +from importlib.util import find_spec from pathlib import Path from typing import TYPE_CHECKING @@ -50,9 +50,7 @@ if not H2_ENABLED: ) ) -try: - import httpx # noqa: F401 -except ImportError: +if not find_spec("httpx"): collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") @@ -130,11 +128,8 @@ def pytest_runtest_setup(item): ] for module in optional_deps: - if item.get_closest_marker(f"requires_{module}"): - try: - importlib.import_module(module) - except ImportError: - pytest.skip(f"{module} is not installed") + if item.get_closest_marker(f"requires_{module}") and find_spec(module) is None: + pytest.skip(f"{module} is not installed") if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None: pytest.skip("mitmdump is not available") diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 13a432476..b99c323c5 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,7 @@ from __future__ import annotations import warnings +from importlib.util import find_spec from itertools import chain from logging import getLogger from typing import TYPE_CHECKING, Any @@ -51,11 +52,7 @@ else: else: ACCEPTED_ENCODINGS.append(b"br") -try: - import zstandard # noqa: F401 -except ImportError: - pass -else: +if find_spec("zstandard") is not None: ACCEPTED_ENCODINGS.append(b"zstd") diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 73f86bc71..2a77ee2ac 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,10 +1,7 @@ """Boto/botocore helpers""" +from importlib.util import find_spec + def is_botocore_available() -> bool: - try: - import botocore # noqa: F401,PLC0415 - - return True - except ImportError: - return False + return find_spec("botocore") is not None diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 311aff7df..ca18c8038 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,4 +1,5 @@ from gzip import GzipFile +from importlib.util import find_spec from io import BytesIO from logging import WARNING from pathlib import Path @@ -65,10 +66,7 @@ def _skip_if_no_br() -> None: def _skip_if_no_zstd() -> None: - try: - import zstandard # noqa: F401,PLC0415 - except ImportError: - pytest.skip("no zstd support (zstandard)") + pytest.importorskip("zstandard") class TestHttpCompression: @@ -172,17 +170,8 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_bytes", 74837) def test_process_response_br_unsupported(self): - try: - try: - import brotli # noqa: F401,PLC0415 - - pytest.skip("Requires not having brotli support") - except ImportError: - import brotlicffi # noqa: F401,PLC0415 - - pytest.skip("Requires not having brotli support") - except ImportError: - pass + if find_spec("brotli") is not None or find_spec("brotlicffi") is not None: + pytest.skip("Requires not having brotli support") response = self._getresponse("br") request = response.request assert response.headers["Content-Encoding"] == b"br" @@ -226,12 +215,8 @@ class TestHttpCompression: assert "Content-Encoding" not in newresponse.headers def test_process_response_zstd_unsupported(self): - try: - import zstandard # noqa: F401,PLC0415 - + if find_spec("zstandard") is not None: pytest.skip("Requires not having zstandard support") - except ImportError: - pass response = self._getresponse("zstd-static-content-size") request = response.request assert response.headers["Content-Encoding"] == b"zstd" diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index 82e2597a5..bdd0d7614 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -436,10 +436,7 @@ class TestS3FeedStorage: class TestGCSFeedStorage: def test_parse_settings(self): - try: - from google.cloud.storage import Client # noqa: F401,PLC0415 - except ImportError: - pytest.skip("GCSFeedStorage requires google-cloud-storage") + pytest.importorskip("google.cloud.storage") settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"} crawler = get_crawler(settings_dict=settings) @@ -450,10 +447,7 @@ class TestGCSFeedStorage: assert storage.blob_name == "export.csv" def test_parse_empty_acl(self): - try: - from google.cloud.storage import Client # noqa: F401,PLC0415 - except ImportError: - pytest.skip("GCSFeedStorage requires google-cloud-storage") + pytest.importorskip("google.cloud.storage") settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} crawler = get_crawler(settings_dict=settings) @@ -467,10 +461,7 @@ class TestGCSFeedStorage: @coroutine_test async def test_store(self): - try: - from google.cloud.storage import Client # noqa: F401,PLC0415 - except ImportError: - pytest.skip("GCSFeedStorage requires google-cloud-storage") + pytest.importorskip("google.cloud.storage") uri = "gs://mybucket/export.csv" project_id = "myproject-123" @@ -492,10 +483,7 @@ class TestGCSFeedStorage: @coroutine_test async def test_store_closes_file_on_upload_error(self): - try: - from google.cloud.storage import Client # noqa: F401,PLC0415 - except ImportError: - pytest.skip("GCSFeedStorage requires google-cloud-storage") + pytest.importorskip("google.cloud.storage") uri = "gs://mybucket/export.csv" project_id = "myproject-123" diff --git a/tests/utils/robotstxt.py b/tests/utils/robotstxt.py index c1b2f28c6..38eaebf41 100644 --- a/tests/utils/robotstxt.py +++ b/tests/utils/robotstxt.py @@ -1,12 +1,7 @@ from __future__ import annotations +from importlib.util import find_spec + def rerp_available() -> bool: - # check if robotexclusionrulesparser is installed - try: - from robotexclusionrulesparser import ( # noqa: PLC0415 - RobotExclusionRulesParser, # noqa: F401 - ) - except ImportError: - return False - return True + return find_spec("robotexclusionrulesparser") is not None