diff --git a/docs/topics/api.rst b/docs/topics/api.rst index b11de2914..d90eb0bad 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -213,6 +213,8 @@ SpiderLoader API :param request: queried request :type request: :class:`~scrapy.Request` instance +.. autoclass:: DummySpiderLoader + .. _topics-api-signals: Signals API diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 68c5079cf..65f2e5ebd 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1868,15 +1868,6 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception But you can choose to silence this exception and turn it into a simple warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. -.. note:: - Some :ref:`scrapy commands ` run with this setting to ``True`` - already (i.e. they will only issue a warning and will not fail) - since they do not actually need to load spider classes to work: - :command:`scrapy runspider `, - :command:`scrapy settings `, - :command:`scrapy startproject `, - :command:`scrapy version `. - .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b08fd3409..81e507a4e 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -201,7 +201,8 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) -> opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) - cmd.crawler_process = CrawlerProcess(settings) + if cmd.requires_crawler_process: + cmd.crawler_process = CrawlerProcess(settings) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 56199cc01..2818ead77 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -19,11 +19,13 @@ if TYPE_CHECKING: from collections.abc import Iterable from scrapy.crawler import Crawler, CrawlerProcess + from scrapy.settings import Settings class ScrapyCommand: requires_project: bool = False - crawler_process: CrawlerProcess | None = None + requires_crawler_process: bool = True + crawler_process: CrawlerProcess | None = None # set in scrapy.cmdline # default settings to be used for this command instead of global defaults default_settings: dict[str, Any] = {} @@ -31,7 +33,7 @@ class ScrapyCommand: exitcode: int = 0 def __init__(self) -> None: - self.settings: Any = None # set in scrapy.cmdline + self.settings: Settings | None = None # set in scrapy.cmdline def set_crawler(self, crawler: Crawler) -> None: if hasattr(self, "_crawler"): @@ -68,6 +70,7 @@ class ScrapyCommand: """ Populate option parse with options available for this command """ + assert self.settings is not None group = parser.add_argument_group(title="Global Options") group.add_argument( "--logfile", metavar="FILE", help="log file. if omitted stderr will be used" @@ -100,6 +103,7 @@ class ScrapyCommand: group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args: list[str], opts: argparse.Namespace) -> None: + assert self.settings is not None try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: @@ -170,6 +174,7 @@ class BaseRunSpiderCommand(ScrapyCommand): except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output or opts.overwrite_output: + assert self.settings is not None feeds = feed_process_params_from_cli( self.settings, opts.output, diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 56dc1ea55..e9ada0fb6 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -69,6 +69,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: # load contracts + assert self.settings is not None contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) conman = ContractsManager(load_object(c) for c in contracts) runner = TextTestRunner(verbosity=2 if opts.verbose else 1) diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index d153a5271..f2d52673a 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -4,10 +4,12 @@ import sys from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import get_spider_loader class Command(ScrapyCommand): requires_project = True + requires_crawler_process = False default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: @@ -30,10 +32,11 @@ class Command(ScrapyCommand): if len(args) != 1: raise UsageError + assert self.settings is not None editor = self.settings["EDITOR"] - assert self.crawler_process + spider_loader = get_spider_loader(self.settings) try: - spidercls = self.crawler_process.spider_loader.load(args[0]) + spidercls = spider_loader.load(args[0]) except KeyError: self._err(f"Spider not found: {args[0]}") return diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 6d4aec3d8..c4abfc4c9 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -11,6 +11,7 @@ from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import get_spider_loader from scrapy.utils.template import render_templatefile, string_camelcase if TYPE_CHECKING: @@ -46,6 +47,7 @@ def verify_url_scheme(url: str) -> str: class Command(ScrapyCommand): requires_project = False + requires_crawler_process = False default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: @@ -92,6 +94,7 @@ class Command(ScrapyCommand): ) def run(self, args: list[str], opts: argparse.Namespace) -> None: + assert self.settings is not None if opts.list: self._list_templates() return @@ -127,6 +130,7 @@ class Command(ScrapyCommand): url: str, template_name: str, ) -> dict[str, Any]: + assert self.settings is not None capitalized_module = "".join(s.capitalize() for s in module.split("_")) return { "project_name": self.settings.get("BOT_NAME"), @@ -147,6 +151,7 @@ class Command(ScrapyCommand): template_file: str | os.PathLike, ) -> None: """Generate the spider module, based on the given template""" + assert self.settings is not None tvars = self._generate_template_variables(module, name, url, template_name) if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) @@ -180,6 +185,7 @@ class Command(ScrapyCommand): print(f" {file.stem}") def _spider_exists(self, name: str) -> bool: + assert self.settings is not None if not self.settings.get("NEWSPIDER_MODULE"): # if run as a standalone command and file with same filename already exists path = Path(name + ".py") @@ -188,12 +194,9 @@ class Command(ScrapyCommand): return True return False - assert self.crawler_process is not None, ( - "crawler_process must be set before calling run" - ) - + spider_loader = get_spider_loader(self.settings) try: - spidercls = self.crawler_process.spider_loader.load(name) + spidercls = spider_loader.load(name) except KeyError: pass else: @@ -215,6 +218,7 @@ class Command(ScrapyCommand): @property def templates_dir(self) -> str: + assert self.settings is not None return str( Path( self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"), diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 3b2f127c2..b4dc97f3d 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -3,6 +3,7 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.commands import ScrapyCommand +from scrapy.spiderloader import get_spider_loader if TYPE_CHECKING: import argparse @@ -10,12 +11,14 @@ if TYPE_CHECKING: class Command(ScrapyCommand): requires_project = True + requires_crawler_process = False default_settings = {"LOG_ENABLED": False} def short_desc(self) -> str: return "List available spiders" def run(self, args: list[str], opts: argparse.Namespace) -> None: - assert self.crawler_process - for s in sorted(self.crawler_process.spider_loader.list()): + assert self.settings is not None + spider_loader = get_spider_loader(self.settings) + for s in sorted(spider_loader.list()): print(s) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 357ca8b37..3e826456e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import DummySpiderLoader from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: @@ -30,7 +31,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType: class Command(BaseRunSpiderCommand): requires_project = False - default_settings = {"SPIDER_LOADER_WARN_ONLY": True} + default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader} def syntax(self) -> str: return "[options] " diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 59f86b9a7..e63031f2d 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): requires_project = False - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: return "[options]" @@ -46,8 +47,8 @@ class Command(ScrapyCommand): ) def run(self, args: list[str], opts: argparse.Namespace) -> None: - assert self.crawler_process - settings = self.crawler_process.settings + assert self.settings is not None + settings = self.settings if opts.get: s = settings.get(opts.get) if isinstance(s, BaseSettings): diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 1adc1530f..323979193 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -34,7 +34,8 @@ def _make_writable(path: Path) -> None: class Command(ScrapyCommand): requires_project = False - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: return " [project_dir]" @@ -132,6 +133,7 @@ class Command(ScrapyCommand): @property def templates_dir(self) -> str: + assert self.settings is not None return str( Path( self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"), diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 713a78ad9..30b0e9fd7 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -6,7 +6,8 @@ from scrapy.utils.versions import get_versions class Command(ScrapyCommand): - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: return "[-v]" diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c22b8603b..8e3223a5c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -5,22 +5,21 @@ import contextlib import logging import pprint import signal -from typing import TYPE_CHECKING, Any, TypeVar, cast +from typing import TYPE_CHECKING, Any, TypeVar from twisted.internet.defer import ( Deferred, DeferredList, inlineCallbacks, ) -from zope.interface.verify import verifyClass from scrapy import Spider, signals from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.extension import ExtensionManager -from scrapy.interfaces import ISpiderLoader -from scrapy.settings import BaseSettings, Settings, overridden_settings +from scrapy.settings import Settings, overridden_settings from scrapy.signalmanager import SignalManager +from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader from scrapy.utils.asyncio import is_asyncio_available from scrapy.utils.defer import deferred_from_coro, deferred_to_future from scrapy.utils.log import ( @@ -46,7 +45,6 @@ if TYPE_CHECKING: from collections.abc import Generator, Iterable from scrapy.logformatter import LogFormatter - from scrapy.spiderloader import SpiderLoaderProtocol from scrapy.statscollectors import StatsCollector from scrapy.utils.request import RequestFingerprinterProtocol @@ -324,22 +322,12 @@ class Crawler: class CrawlerRunnerBase: - @staticmethod - def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: - """Get SpiderLoader instance from settings""" - cls_path = settings.get("SPIDER_LOADER_CLASS") - loader_cls = load_object(cls_path) - verifyClass(ISpiderLoader, loader_cls) - return cast( - "SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()) - ) - def __init__(self, settings: dict[str, Any] | Settings | None = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) AddonManager.load_pre_crawler_settings(settings) self.settings: Settings = settings - self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings) + self.spider_loader: SpiderLoaderProtocol = get_spider_loader(settings) self._crawlers: set[Crawler] = set() self.bootstrap_failed = False diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index f537e0593..8eac188c8 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,12 +3,13 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from typing import TYPE_CHECKING, Protocol +from typing import TYPE_CHECKING, Protocol, cast from zope.interface import implementer +from zope.interface.verify import verifyClass from scrapy.interfaces import ISpiderLoader -from scrapy.utils.misc import walk_modules +from scrapy.utils.misc import load_object, walk_modules from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: @@ -21,6 +22,14 @@ if TYPE_CHECKING: from scrapy.settings import BaseSettings +def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: + """Get SpiderLoader instance from settings""" + cls_path = settings.get("SPIDER_LOADER_CLASS") + loader_cls = load_object(cls_path) + verifyClass(ISpiderLoader, loader_cls) + return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())) + + class SpiderLoaderProtocol(Protocol): @classmethod def from_settings(cls, settings: BaseSettings) -> Self: @@ -120,3 +129,21 @@ class SpiderLoader: Return a list with the names of all spiders available in the project. """ return list(self._spiders.keys()) + + +@implementer(ISpiderLoader) +class DummySpiderLoader: + """A dummy spider loader that does not load any spiders.""" + + @classmethod + def from_settings(cls, settings: BaseSettings) -> Self: + return cls() + + def load(self, spider_name: str) -> type[Spider]: + raise KeyError("DummySpiderLoader doesn't load any spiders") + + def list(self) -> list[str]: + return [] + + def find_by_request(self, request: Request) -> __builtins__.list[str]: + return [] diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ce5963cd7..56cb21650 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -30,7 +30,6 @@ from scrapy.crawler import ( from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings -from scrapy.spiderloader import SpiderLoader from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_coro from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider @@ -570,10 +569,6 @@ class SpiderLoaderWithWrongInterface: pass -class CustomSpiderLoader(SpiderLoader): - pass - - class TestCrawlerRunner(TestBaseCrawler): def test_spider_manager_verify_interface(self): settings = Settings( diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 476487a04..245507c0b 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,10 +1,8 @@ import contextlib import shutil import sys -import tempfile import warnings from pathlib import Path -from tempfile import mkdtemp from unittest import mock import pytest @@ -17,7 +15,7 @@ from scrapy.crawler import CrawlerRunner from scrapy.http import Request from scrapy.interfaces import ISpiderLoader from scrapy.settings import Settings -from scrapy.spiderloader import SpiderLoader +from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader module_dir = Path(__file__).resolve().parent @@ -27,73 +25,76 @@ def _copytree(source: Path, target: Path): shutil.copytree(source, target) +@pytest.fixture +def spider_loader_env(tmp_path): + orig_spiders_dir = module_dir / "test_spiders" + spiders_dir = tmp_path / "test_spiders_xxx" + _copytree(orig_spiders_dir, spiders_dir) + sys.path.append(str(tmp_path)) + settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) + + yield settings, spiders_dir + + sys.modules.pop("test_spiders_xxx", None) + sys.path.remove(str(tmp_path)) + + +@pytest.fixture +def spider_loader(spider_loader_env): + settings, _ = spider_loader_env + return SpiderLoader.from_settings(settings) + + class TestSpiderLoader: - def setup_method(self): - orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(tempfile.mkdtemp()) - self.spiders_dir = self.tmpdir / "test_spiders_xxx" - _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(str(self.tmpdir)) - settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) - self.spider_loader = SpiderLoader.from_settings(settings) + def test_interface(self, spider_loader): + verifyObject(ISpiderLoader, spider_loader) - def teardown_method(self): - del self.spider_loader - del sys.modules["test_spiders_xxx"] - sys.path.remove(str(self.tmpdir)) - - def test_interface(self): - verifyObject(ISpiderLoader, self.spider_loader) - - def test_list(self): - assert set(self.spider_loader.list()) == { + def test_list(self, spider_loader): + assert set(spider_loader.list()) == { "spider1", "spider2", "spider3", "spider4", } - def test_load(self): - spider1 = self.spider_loader.load("spider1") + def test_load(self, spider_loader): + spider1 = spider_loader.load("spider1") assert spider1.__name__ == "Spider1" - def test_find_by_request(self): - assert self.spider_loader.find_by_request( - Request("http://scrapy1.org/test") - ) == ["spider1"] - assert self.spider_loader.find_by_request( - Request("http://scrapy2.org/test") - ) == ["spider2"] + def test_find_by_request(self, spider_loader): + assert spider_loader.find_by_request(Request("http://scrapy1.org/test")) == [ + "spider1" + ] + assert spider_loader.find_by_request(Request("http://scrapy2.org/test")) == [ + "spider2" + ] assert set( - self.spider_loader.find_by_request(Request("http://scrapy3.org/test")) + spider_loader.find_by_request(Request("http://scrapy3.org/test")) ) == {"spider1", "spider2"} - assert ( - self.spider_loader.find_by_request(Request("http://scrapy999.org/test")) - == [] - ) - assert self.spider_loader.find_by_request(Request("http://spider3.com")) == [] - assert self.spider_loader.find_by_request( + assert spider_loader.find_by_request(Request("http://scrapy999.org/test")) == [] + assert spider_loader.find_by_request(Request("http://spider3.com")) == [] + assert spider_loader.find_by_request( Request("http://spider3.com/onlythis") ) == ["spider3"] def test_load_spider_module(self): module = "tests.test_spiderloader.test_spiders.spider1" settings = Settings({"SPIDER_MODULES": [module]}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 1 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 1 def test_load_spider_module_multiple(self): prefix = "tests.test_spiderloader.test_spiders." module = ",".join(prefix + s for s in ("spider1", "spider2")) settings = Settings({"SPIDER_MODULES": module}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 2 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 2 def test_load_base_spider(self): module = "tests.test_spiderloader.test_spiders.spider0" settings = Settings({"SPIDER_MODULES": [module]}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 0 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 0 def test_load_spider_module_from_addons(self): module = "tests.test_spiderloader.spiders_from_addons.spider0" @@ -183,27 +184,14 @@ class TestSpiderLoader: class TestDuplicateSpiderNameLoader: - def setup_method(self): - orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(mkdtemp()) - self.spiders_dir = self.tmpdir / "test_spiders_xxx" - _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(str(self.tmpdir)) - self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) + def test_dupename_warning(self, spider_loader_env): + settings, spiders_dir = spider_loader_env - def teardown_method(self): - del sys.modules["test_spiders_xxx"] - sys.path.remove(str(self.tmpdir)) - - def test_dupename_warning(self): # copy 1 spider module so as to have duplicate spider name - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider3.py", - self.tmpdir / "test_spiders_xxx" / "spider3dupe.py", - ) + shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py") with warnings.catch_warnings(record=True) as w: - spider_loader = SpiderLoader.from_settings(self.settings) + spider_loader = SpiderLoader.from_settings(settings) assert len(w) == 1 msg = str(w[0].message) @@ -218,20 +206,15 @@ class TestDuplicateSpiderNameLoader: spiders = set(spider_loader.list()) assert spiders == {"spider1", "spider2", "spider3", "spider4"} - def test_multiple_dupename_warning(self): + def test_multiple_dupename_warning(self, spider_loader_env): + settings, spiders_dir = spider_loader_env # copy 2 spider modules so as to have duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider1.py", - self.tmpdir / "test_spiders_xxx" / "spider1dupe.py", - ) - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider2.py", - self.tmpdir / "test_spiders_xxx" / "spider2dupe.py", - ) + shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py") + shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py") with warnings.catch_warnings(record=True) as w: - spider_loader = SpiderLoader.from_settings(self.settings) + spider_loader = SpiderLoader.from_settings(settings) assert len(w) == 1 msg = str(w[0].message) @@ -247,3 +230,25 @@ class TestDuplicateSpiderNameLoader: spiders = set(spider_loader.list()) assert spiders == {"spider1", "spider2", "spider3", "spider4"} + + +class CustomSpiderLoader(SpiderLoader): + pass + + +def test_custom_spider_loader(): + settings = Settings( + { + "SPIDER_LOADER_CLASS": CustomSpiderLoader, + } + ) + spider_loader = get_spider_loader(settings) + assert isinstance(spider_loader, CustomSpiderLoader) + + +def test_dummy_spider_loader(spider_loader_env): + settings, _ = spider_loader_env + spider_loader = DummySpiderLoader.from_settings(settings) + assert not spider_loader.list() + with pytest.raises(KeyError): + spider_loader.load("spider1")