Add SpiderLoaderProtocol. (#6694)

scrapy/crawler.py

@@ -42,7 +42,7 @@ if TYPE_CHECKING:
     from collections.abc import Generator, Iterable
 
     from scrapy.logformatter import LogFormatter
-    from scrapy.spiderloader import SpiderLoader
+    from scrapy.spiderloader import SpiderLoaderProtocol
     from scrapy.statscollectors import StatsCollector
     from scrapy.utils.request import RequestFingerprinterProtocol
 
@@ -282,19 +282,21 @@ class CrawlerRunner:
     )
 
     @staticmethod
-    def _get_spider_loader(settings: BaseSettings) -> SpiderLoader:
+    def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
         """Get SpiderLoader instance from settings"""
         cls_path = settings.get("SPIDER_LOADER_CLASS")
         loader_cls = load_object(cls_path)
         verifyClass(ISpiderLoader, loader_cls)
-        return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy()))
+        return cast(
+            "SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())
+        )
 
     def __init__(self, settings: dict[str, Any] | Settings | None = None):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
         AddonManager.load_pre_crawler_settings(settings)
         self.settings: Settings = settings
-        self.spider_loader: SpiderLoader = self._get_spider_loader(settings)
+        self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings)
         self._crawlers: set[Crawler] = set()
         self._active: set[Deferred[None]] = set()
         self.bootstrap_failed = False

scrapy/spiderloader.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import traceback
 import warnings
 from collections import defaultdict
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Protocol
 
 from zope.interface import implementer
 
@@ -21,6 +21,23 @@ if TYPE_CHECKING:
     from scrapy.settings import BaseSettings
 
 
+class SpiderLoaderProtocol(Protocol):
+    @classmethod
+    def from_settings(cls, settings: BaseSettings) -> Self:
+        """Return an instance of the class for the given settings"""
+
+    def load(self, spider_name: str) -> type[Spider]:
+        """Return the Spider class for the given spider name. If the spider
+        name is not found, it must raise a KeyError."""
+
+    def list(self) -> list[str]:
+        """Return a list with the names of all spiders available in the
+        project"""
+
+    def find_by_request(self, request: Request) -> __builtins__.list[str]:
+        """Return the list of spiders names that can handle the given request"""
+
+
 @implementer(ISpiderLoader)
 class SpiderLoader:
     """

scrapy/utils/spider.py

@@ -15,7 +15,7 @@ if TYPE_CHECKING:
     from twisted.internet.defer import Deferred
 
     from scrapy import Request
-    from scrapy.spiderloader import SpiderLoader
+    from scrapy.spiderloader import SpiderLoaderProtocol
 
 
 logger = logging.getLogger(__name__)
@@ -64,7 +64,7 @@ def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]:
 
 @overload
 def spidercls_for_request(
-    spider_loader: SpiderLoader,
+    spider_loader: SpiderLoaderProtocol,
     request: Request,
     default_spidercls: type[Spider],
     log_none: bool = ...,
@@ -74,7 +74,7 @@ def spidercls_for_request(
 
 @overload
 def spidercls_for_request(
-    spider_loader: SpiderLoader,
+    spider_loader: SpiderLoaderProtocol,
     request: Request,
     default_spidercls: Literal[None],
     log_none: bool = ...,
@@ -84,7 +84,7 @@ def spidercls_for_request(
 
 @overload
 def spidercls_for_request(
-    spider_loader: SpiderLoader,
+    spider_loader: SpiderLoaderProtocol,
     request: Request,
     *,
     log_none: bool = ...,
@@ -93,7 +93,7 @@ def spidercls_for_request(
 
 
 def spidercls_for_request(
-    spider_loader: SpiderLoader,
+    spider_loader: SpiderLoaderProtocol,
     request: Request,
     default_spidercls: type[Spider] | None = None,
     log_none: bool = False,