RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Don't use CrawlerProcess in the commands that don't need it. (#6824) 

* Don't use CrawlerProcess in the commands that don't need it.

* Use a dummy spider loader in runspider.
This commit is contained in:
Andrey Rakhmatullin 2025-05-28 19:57:33 +05:00 committed by GitHub
parent 05b3b205ce
commit e0b9f2d8f6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 150 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/topics/api.rst
View File

@ -213,6 +213,8 @@ SpiderLoader API
:param request: queried request
:type request: :class:`~scrapy.Request` instance
.. autoclass:: DummySpiderLoader
.. _topics-api-signals:
Signals API

9
docs/topics/settings.rst
View File

@ -1868,15 +1868,6 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception
But you can choose to silence this exception and turn it into a simple
warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
.. note::
Some :ref:`scrapy commands <topics-commands>` run with this setting to ``True``
already (i.e. they will only issue a warning and will not fail)
since they do not actually need to load spider classes to work:
:command:`scrapy runspider <runspider>`,
:command:`scrapy settings <settings>`,
:command:`scrapy startproject <startproject>`,
:command:`scrapy version <version>`.
.. setting:: SPIDER_MIDDLEWARES
SPIDER_MIDDLEWARES

3
scrapy/cmdline.py
View File

@ -201,7 +201,8 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) ->
opts, args = parser.parse_known_args(args=argv[1:])
_run_print_help(parser, cmd.process_options, args, opts)
cmd.crawler_process = CrawlerProcess(settings)
if cmd.requires_crawler_process:
cmd.crawler_process = CrawlerProcess(settings)
_run_print_help(parser, _run_command, cmd, args, opts)
sys.exit(cmd.exitcode)

9
scrapy/commands/__init__.py
View File

@ -19,11 +19,13 @@ if TYPE_CHECKING:
from collections.abc import Iterable
from scrapy.crawler import Crawler, CrawlerProcess
from scrapy.settings import Settings
class ScrapyCommand:
requires_project: bool = False
crawler_process: CrawlerProcess | None = None
requires_crawler_process: bool = True
crawler_process: CrawlerProcess | None = None # set in scrapy.cmdline
# default settings to be used for this command instead of global defaults
default_settings: dict[str, Any] = {}
@ -31,7 +33,7 @@ class ScrapyCommand:
exitcode: int = 0
def __init__(self) -> None:
self.settings: Any = None # set in scrapy.cmdline
self.settings: Settings | None = None # set in scrapy.cmdline
def set_crawler(self, crawler: Crawler) -> None:
if hasattr(self, "_crawler"):
@ -68,6 +70,7 @@ class ScrapyCommand:
"""
Populate option parse with options available for this command
"""
assert self.settings is not None
group = parser.add_argument_group(title="Global Options")
group.add_argument(
"--logfile", metavar="FILE", help="log file. if omitted stderr will be used"
@ -100,6 +103,7 @@ class ScrapyCommand:
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
def process_options(self, args: list[str], opts: argparse.Namespace) -> None:
assert self.settings is not None
try:
self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
except ValueError:
@ -170,6 +174,7 @@ class BaseRunSpiderCommand(ScrapyCommand):
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output or opts.overwrite_output:
assert self.settings is not None
feeds = feed_process_params_from_cli(
self.settings,
opts.output,

1
scrapy/commands/check.py
View File

@ -69,6 +69,7 @@ class Command(ScrapyCommand):
def run(self, args: list[str], opts: argparse.Namespace) -> None:
# load contracts
assert self.settings is not None
contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS"))
conman = ContractsManager(load_object(c) for c in contracts)
runner = TextTestRunner(verbosity=2 if opts.verbose else 1)

7
scrapy/commands/edit.py
View File

@ -4,10 +4,12 @@ import sys
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.spiderloader import get_spider_loader
class Command(ScrapyCommand):
requires_project = True
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
def syntax(self) -> str:
@ -30,10 +32,11 @@ class Command(ScrapyCommand):
if len(args) != 1:
raise UsageError
assert self.settings is not None
editor = self.settings["EDITOR"]
assert self.crawler_process
spider_loader = get_spider_loader(self.settings)
try:
spidercls = self.crawler_process.spider_loader.load(args[0])
spidercls = spider_loader.load(args[0])
except KeyError:
self._err(f"Spider not found: {args[0]}")
return

14
scrapy/commands/genspider.py
View File

@ -11,6 +11,7 @@ from urllib.parse import urlparse
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.spiderloader import get_spider_loader
from scrapy.utils.template import render_templatefile, string_camelcase
if TYPE_CHECKING:
@ -46,6 +47,7 @@ def verify_url_scheme(url: str) -> str:
class Command(ScrapyCommand):
requires_project = False
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
def syntax(self) -> str:
@ -92,6 +94,7 @@ class Command(ScrapyCommand):
)
def run(self, args: list[str], opts: argparse.Namespace) -> None:
assert self.settings is not None
if opts.list:
self._list_templates()
return
@ -127,6 +130,7 @@ class Command(ScrapyCommand):
url: str,
template_name: str,
) -> dict[str, Any]:
assert self.settings is not None
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
return {
"project_name": self.settings.get("BOT_NAME"),
@ -147,6 +151,7 @@ class Command(ScrapyCommand):
template_file: str | os.PathLike,
) -> None:
"""Generate the spider module, based on the given template"""
assert self.settings is not None
tvars = self._generate_template_variables(module, name, url, template_name)
if self.settings.get("NEWSPIDER_MODULE"):
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
@ -180,6 +185,7 @@ class Command(ScrapyCommand):
print(f" {file.stem}")
def _spider_exists(self, name: str) -> bool:
assert self.settings is not None
if not self.settings.get("NEWSPIDER_MODULE"):
# if run as a standalone command and file with same filename already exists
path = Path(name + ".py")
@ -188,12 +194,9 @@ class Command(ScrapyCommand):
return True
return False
assert self.crawler_process is not None, (
"crawler_process must be set before calling run"
)
spider_loader = get_spider_loader(self.settings)
try:
spidercls = self.crawler_process.spider_loader.load(name)
spidercls = spider_loader.load(name)
except KeyError:
pass
else:
@ -215,6 +218,7 @@ class Command(ScrapyCommand):
@property
def templates_dir(self) -> str:
assert self.settings is not None
return str(
Path(
self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),

7
scrapy/commands/list.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy.commands import ScrapyCommand
from scrapy.spiderloader import get_spider_loader
if TYPE_CHECKING:
import argparse
@ -10,12 +11,14 @@ if TYPE_CHECKING:
class Command(ScrapyCommand):
requires_project = True
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
def short_desc(self) -> str:
return "List available spiders"
def run(self, args: list[str], opts: argparse.Namespace) -> None:
assert self.crawler_process
for s in sorted(self.crawler_process.spider_loader.list()):
assert self.settings is not None
spider_loader = get_spider_loader(self.settings)
for s in sorted(spider_loader.list()):
print(s)

3
scrapy/commands/runspider.py
View File

@ -7,6 +7,7 @@ from typing import TYPE_CHECKING
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.spiderloader import DummySpiderLoader
from scrapy.utils.spider import iter_spider_classes
if TYPE_CHECKING:
@ -30,7 +31,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType:
class Command(BaseRunSpiderCommand):
requires_project = False
default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader}
def syntax(self) -> str:
return "[options] <spider_file>"

7
scrapy/commands/settings.py
View File

@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
requires_project = False
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "[options]"
@ -46,8 +47,8 @@ class Command(ScrapyCommand):
)
def run(self, args: list[str], opts: argparse.Namespace) -> None:
assert self.crawler_process
settings = self.crawler_process.settings
assert self.settings is not None
settings = self.settings
if opts.get:
s = settings.get(opts.get)
if isinstance(s, BaseSettings):

4
scrapy/commands/startproject.py
View File

@ -34,7 +34,8 @@ def _make_writable(path: Path) -> None:
class Command(ScrapyCommand):
requires_project = False
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "<project_name> [project_dir]"
@ -132,6 +133,7 @@ class Command(ScrapyCommand):
@property
def templates_dir(self) -> str:
assert self.settings is not None
return str(
Path(
self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),

3
scrapy/commands/version.py
View File

@ -6,7 +6,8 @@ from scrapy.utils.versions import get_versions
class Command(ScrapyCommand):
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "[-v]"

20
scrapy/crawler.py
View File

@ -5,22 +5,21 @@ import contextlib
import logging
import pprint
import signal
from typing import TYPE_CHECKING, Any, TypeVar, cast
from typing import TYPE_CHECKING, Any, TypeVar
from twisted.internet.defer import (
Deferred,
DeferredList,
inlineCallbacks,
)
from zope.interface.verify import verifyClass
from scrapy import Spider, signals
from scrapy.addons import AddonManager
from scrapy.core.engine import ExecutionEngine
from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
from scrapy.settings import BaseSettings, Settings, overridden_settings
from scrapy.settings import Settings, overridden_settings
from scrapy.signalmanager import SignalManager
from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.defer import deferred_from_coro, deferred_to_future
from scrapy.utils.log import (
@ -46,7 +45,6 @@ if TYPE_CHECKING:
from collections.abc import Generator, Iterable
from scrapy.logformatter import LogFormatter
from scrapy.spiderloader import SpiderLoaderProtocol
from scrapy.statscollectors import StatsCollector
from scrapy.utils.request import RequestFingerprinterProtocol
@ -324,22 +322,12 @@ class Crawler:
class CrawlerRunnerBase:
@staticmethod
def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
"""Get SpiderLoader instance from settings"""
cls_path = settings.get("SPIDER_LOADER_CLASS")
loader_cls = load_object(cls_path)
verifyClass(ISpiderLoader, loader_cls)
return cast(
"SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())
)
def __init__(self, settings: dict[str, Any] | Settings | None = None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
AddonManager.load_pre_crawler_settings(settings)
self.settings: Settings = settings
self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings)
self.spider_loader: SpiderLoaderProtocol = get_spider_loader(settings)
self._crawlers: set[Crawler] = set()
self.bootstrap_failed = False

31
scrapy/spiderloader.py
View File

@ -3,12 +3,13 @@ from __future__ import annotations
import traceback
import warnings
from collections import defaultdict
from typing import TYPE_CHECKING, Protocol
from typing import TYPE_CHECKING, Protocol, cast
from zope.interface import implementer
from zope.interface.verify import verifyClass
from scrapy.interfaces import ISpiderLoader
from scrapy.utils.misc import walk_modules
from scrapy.utils.misc import load_object, walk_modules
from scrapy.utils.spider import iter_spider_classes
if TYPE_CHECKING:
@ -21,6 +22,14 @@ if TYPE_CHECKING:
from scrapy.settings import BaseSettings
def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
"""Get SpiderLoader instance from settings"""
cls_path = settings.get("SPIDER_LOADER_CLASS")
loader_cls = load_object(cls_path)
verifyClass(ISpiderLoader, loader_cls)
return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()))
class SpiderLoaderProtocol(Protocol):
@classmethod
def from_settings(cls, settings: BaseSettings) -> Self:
@ -120,3 +129,21 @@ class SpiderLoader:
Return a list with the names of all spiders available in the project.
"""
return list(self._spiders.keys())
@implementer(ISpiderLoader)
class DummySpiderLoader:
"""A dummy spider loader that does not load any spiders."""
@classmethod
def from_settings(cls, settings: BaseSettings) -> Self:
return cls()
def load(self, spider_name: str) -> type[Spider]:
raise KeyError("DummySpiderLoader doesn't load any spiders")
def list(self) -> list[str]:
return []
def find_by_request(self, request: Request) -> __builtins__.list[str]:
return []

5
tests/test_crawler.py
View File

@ -30,7 +30,6 @@ from scrapy.crawler import (
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.throttle import AutoThrottle
from scrapy.settings import Settings, default_settings
from scrapy.spiderloader import SpiderLoader
from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_coro
from scrapy.utils.log import configure_logging, get_scrapy_root_handler
from scrapy.utils.spider import DefaultSpider
@ -570,10 +569,6 @@ class SpiderLoaderWithWrongInterface:
pass
class CustomSpiderLoader(SpiderLoader):
pass
class TestCrawlerRunner(TestBaseCrawler):
def test_spider_manager_verify_interface(self):
settings = Settings(

145
tests/test_spiderloader/__init__.py
View File

@ -1,10 +1,8 @@
import contextlib
import shutil
import sys
import tempfile
import warnings
from pathlib import Path
from tempfile import mkdtemp
from unittest import mock
import pytest
@ -17,7 +15,7 @@ from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
from scrapy.interfaces import ISpiderLoader
from scrapy.settings import Settings
from scrapy.spiderloader import SpiderLoader
from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader
module_dir = Path(__file__).resolve().parent
@ -27,73 +25,76 @@ def _copytree(source: Path, target: Path):
shutil.copytree(source, target)
@pytest.fixture
def spider_loader_env(tmp_path):
orig_spiders_dir = module_dir / "test_spiders"
spiders_dir = tmp_path / "test_spiders_xxx"
_copytree(orig_spiders_dir, spiders_dir)
sys.path.append(str(tmp_path))
settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
yield settings, spiders_dir
sys.modules.pop("test_spiders_xxx", None)
sys.path.remove(str(tmp_path))
@pytest.fixture
def spider_loader(spider_loader_env):
settings, _ = spider_loader_env
return SpiderLoader.from_settings(settings)
class TestSpiderLoader:
def setup_method(self):
orig_spiders_dir = module_dir / "test_spiders"
self.tmpdir = Path(tempfile.mkdtemp())
self.spiders_dir = self.tmpdir / "test_spiders_xxx"
_copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(str(self.tmpdir))
settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
self.spider_loader = SpiderLoader.from_settings(settings)
def test_interface(self, spider_loader):
verifyObject(ISpiderLoader, spider_loader)
def teardown_method(self):
del self.spider_loader
del sys.modules["test_spiders_xxx"]
sys.path.remove(str(self.tmpdir))
def test_interface(self):
verifyObject(ISpiderLoader, self.spider_loader)
def test_list(self):
assert set(self.spider_loader.list()) == {
def test_list(self, spider_loader):
assert set(spider_loader.list()) == {
"spider1",
"spider2",
"spider3",
"spider4",
}
def test_load(self):
spider1 = self.spider_loader.load("spider1")
def test_load(self, spider_loader):
spider1 = spider_loader.load("spider1")
assert spider1.__name__ == "Spider1"
def test_find_by_request(self):
assert self.spider_loader.find_by_request(
Request("http://scrapy1.org/test")
) == ["spider1"]
assert self.spider_loader.find_by_request(
Request("http://scrapy2.org/test")
) == ["spider2"]
def test_find_by_request(self, spider_loader):
assert spider_loader.find_by_request(Request("http://scrapy1.org/test")) == [
"spider1"
]
assert spider_loader.find_by_request(Request("http://scrapy2.org/test")) == [
"spider2"
]
assert set(
self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))
spider_loader.find_by_request(Request("http://scrapy3.org/test"))
) == {"spider1", "spider2"}
assert (
self.spider_loader.find_by_request(Request("http://scrapy999.org/test"))
== []
)
assert self.spider_loader.find_by_request(Request("http://spider3.com")) == []
assert self.spider_loader.find_by_request(
assert spider_loader.find_by_request(Request("http://scrapy999.org/test")) == []
assert spider_loader.find_by_request(Request("http://spider3.com")) == []
assert spider_loader.find_by_request(
Request("http://spider3.com/onlythis")
) == ["spider3"]
def test_load_spider_module(self):
module = "tests.test_spiderloader.test_spiders.spider1"
settings = Settings({"SPIDER_MODULES": [module]})
self.spider_loader = SpiderLoader.from_settings(settings)
assert len(self.spider_loader._spiders) == 1
spider_loader = SpiderLoader.from_settings(settings)
assert len(spider_loader._spiders) == 1
def test_load_spider_module_multiple(self):
prefix = "tests.test_spiderloader.test_spiders."
module = ",".join(prefix + s for s in ("spider1", "spider2"))
settings = Settings({"SPIDER_MODULES": module})
self.spider_loader = SpiderLoader.from_settings(settings)
assert len(self.spider_loader._spiders) == 2
spider_loader = SpiderLoader.from_settings(settings)
assert len(spider_loader._spiders) == 2
def test_load_base_spider(self):
module = "tests.test_spiderloader.test_spiders.spider0"
settings = Settings({"SPIDER_MODULES": [module]})
self.spider_loader = SpiderLoader.from_settings(settings)
assert len(self.spider_loader._spiders) == 0
spider_loader = SpiderLoader.from_settings(settings)
assert len(spider_loader._spiders) == 0
def test_load_spider_module_from_addons(self):
module = "tests.test_spiderloader.spiders_from_addons.spider0"
@ -183,27 +184,14 @@ class TestSpiderLoader:
class TestDuplicateSpiderNameLoader:
def setup_method(self):
orig_spiders_dir = module_dir / "test_spiders"
self.tmpdir = Path(mkdtemp())
self.spiders_dir = self.tmpdir / "test_spiders_xxx"
_copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(str(self.tmpdir))
self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
def test_dupename_warning(self, spider_loader_env):
settings, spiders_dir = spider_loader_env
def teardown_method(self):
del sys.modules["test_spiders_xxx"]
sys.path.remove(str(self.tmpdir))
def test_dupename_warning(self):
# copy 1 spider module so as to have duplicate spider name
shutil.copyfile(
self.tmpdir / "test_spiders_xxx" / "spider3.py",
self.tmpdir / "test_spiders_xxx" / "spider3dupe.py",
)
shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py")
with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings)
spider_loader = SpiderLoader.from_settings(settings)
assert len(w) == 1
msg = str(w[0].message)
@ -218,20 +206,15 @@ class TestDuplicateSpiderNameLoader:
spiders = set(spider_loader.list())
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
def test_multiple_dupename_warning(self):
def test_multiple_dupename_warning(self, spider_loader_env):
settings, spiders_dir = spider_loader_env
# copy 2 spider modules so as to have duplicate spider name
# This should issue 2 warning, 1 for each duplicate spider name
shutil.copyfile(
self.tmpdir / "test_spiders_xxx" / "spider1.py",
self.tmpdir / "test_spiders_xxx" / "spider1dupe.py",
)
shutil.copyfile(
self.tmpdir / "test_spiders_xxx" / "spider2.py",
self.tmpdir / "test_spiders_xxx" / "spider2dupe.py",
)
shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py")
shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py")
with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings)
spider_loader = SpiderLoader.from_settings(settings)
assert len(w) == 1
msg = str(w[0].message)
@ -247,3 +230,25 @@ class TestDuplicateSpiderNameLoader:
spiders = set(spider_loader.list())
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
class CustomSpiderLoader(SpiderLoader):
pass
def test_custom_spider_loader():
settings = Settings(
{
"SPIDER_LOADER_CLASS": CustomSpiderLoader,
}
)
spider_loader = get_spider_loader(settings)
assert isinstance(spider_loader, CustomSpiderLoader)
def test_dummy_spider_loader(spider_loader_env):
settings, _ = spider_loader_env
spider_loader = DummySpiderLoader.from_settings(settings)
assert not spider_loader.list()
with pytest.raises(KeyError):
spider_loader.load("spider1")