diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 50da4593a..bbfadc45c 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -341,8 +341,8 @@ list * Syntax: ``scrapy list`` * Requires project: *yes* -List all available spiders in the current project. The output is one spider per -line. +List all :ref:`spiders ` available in the current project. The +output is one spider per line. Usage example:: @@ -350,6 +350,9 @@ Usage example:: spider1 spider2 +Which spiders are listed depends on the configured +:setting:`SPIDER_LOADER_CLASS`. + .. command:: edit edit diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e58815fad..dd1094db4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2083,6 +2083,32 @@ The class that will be used for loading spiders, which must implement the .. note:: This is a :ref:`pre-crawler setting `. +.. setting:: SPIDER_LOADER_REQUIRE_NAME + +SPIDER_LOADER_REQUIRE_NAME +-------------------------- + +Default: ``True`` + +.. note:: + + While the default value is ``True`` for historical reasons, this option is + disabled by default in the ``settings.py`` file generated by the + :command:`startproject` command. + +By default, when loading spiders, Scrapy only loads +:class:`~scrapy.spiders.Spider` subclasses that have a non-empty +:attr:`~scrapy.Spider.name` unless they are decorated with +:func:`~scrapy.spiders.ignore_spider`. + +If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``False``, Scrapy loads all +:class:`~scrapy.spiders.Spider` subclasses unless they are decorated with +:func:`~scrapy.spiders.ignore_spider`. If they do not have a non-empty +:attr:`~scrapy.Spider.name`, their fully-qualified class name is used +as a name. + +.. note:: This is a :ref:`pre-crawler setting `. + .. setting:: SPIDER_LOADER_WARN_ONLY SPIDER_LOADER_WARN_ONLY diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 95c80d5dc..dfee5ec5d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -40,11 +40,16 @@ scrapy.Spider .. attribute:: name - A string which defines the name for this spider. The spider name is how - the spider is located (and instantiated) by Scrapy, so it must be - unique. However, nothing prevents you from instantiating more than one - instance of the same spider. This is the most important spider attribute - and it's required. + A string which defines the name for this spider. + + If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` and you use the + default Scrapy spider loader (see :setting:`SPIDER_LOADER_CLASS`), a + non-empty name is required for the spider to be discoverable by the + Scrapy commands :command:`crawl` and :command:`list`. + + The spider name must be unique to one spider class. If two or more + spiders have the same name, Scrapy commands :command:`crawl` and + :command:`runspider` will only be able to run one of the spiders. If the spider scrapes a single domain, a common practice is to name the spider after the domain, with or without the `TLD`_. So, for example, a @@ -1002,3 +1007,29 @@ Combine SitemapSpider with other sources of urls: .. _robots.txt: https://www.robotstxt.org/ .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/ + + +Base spiders +============ + +Base spiders are :class:`~scrapy.spiders.Spider` subclasses that are not meant +to be run by Scrapy. They are only meant to be subclassed to create regular +spiders or other base spiders. They are one way to share code between two or +more spiders. + +Use the :func:`~scrapy.spiders.ignore_spider` decorator to mark any base spider +class: + +.. autodecorator:: scrapy.spiders.ignore_spider + +For example:: + + from scrapy.spiders import ignore_spider, Spider + + @ignore_spider + class MyBaseSpider(Spider): + pass + +If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` (default), any +:class:`~scrapy.spiders.Spider` subclass without a ``name`` class attribute is +also ignored. diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index e6d5ff96a..a8c33d74f 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -38,8 +38,6 @@ class ScrapyArgumentParser(argparse.ArgumentParser): def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]: - # TODO: add `name` attribute to commands and merge this function with - # scrapy.utils.spider.iter_spider_classes for module in walk_modules_iter(module_name): for obj in vars(module).values(): if ( diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 2113c19d2..8d7eca877 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -98,7 +98,7 @@ class Command(ScrapyCommand): tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: for method in tested_methods: - contract_reqs[spidercls.name].append(method) + contract_reqs[spidername].append(method) elif tested_methods: self.crawler_process.crawl(spidercls) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 9cdb393ab..d24b85918 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -53,10 +53,14 @@ class Command(BaseRunSpiderCommand): module = _import_file(filename) except (ImportError, ValueError) as e: raise UsageError(f"Unable to load {str(filename)!r}: {e}\n") from e - spclasses = list(iter_spider_classes(module)) + # The spider is looked up by file name, so it does not need a name of + # its own. Named spiders still win over nameless ones, which in a file + # with both are usually base spiders. + spclasses = list(iter_spider_classes(module, require_name=False)) if not spclasses: raise UsageError(f"No spider found in file: {filename}\n") - spidercls = spclasses.pop() + named = [spcls for spcls in spclasses if getattr(spcls, "name", None)] + spidercls = (named or spclasses).pop() assert self.crawler_process self.crawler_process.crawl(spidercls, **opts.spargs) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b97844103..c1a7280ad 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -550,6 +550,7 @@ SPIDER_CONTRACTS_BASE = { } SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader" +SPIDER_LOADER_REQUIRE_NAME = True SPIDER_LOADER_WARN_ONLY = False SPIDER_MIDDLEWARES = {} diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 244b7338f..199f2d859 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -55,6 +55,7 @@ class SpiderLoader: """ def __init__(self, settings: BaseSettings): + self.require_name: bool = settings.getbool("SPIDER_LOADER_REQUIRE_NAME") self.spider_modules: list[str] = settings.getlist("SPIDER_MODULES") self.warn_only: bool = settings.getbool("SPIDER_LOADER_WARN_ONLY") self._spiders: dict[str, type[Spider]] = {} @@ -82,9 +83,10 @@ class SpiderLoader: ) def _load_spiders(self, module: ModuleType) -> None: - for spcls in iter_spider_classes(module): - self._found[spcls.name].append((module.__name__, spcls.__name__)) - self._spiders[spcls.name] = spcls + for spcls in iter_spider_classes(module, require_name=self.require_name): + name = spcls._default_name() + self._found[name].append((module.__name__, spcls.__name__)) + self._spiders[name] = spcls def _load_all_spiders(self) -> None: for name in self.spider_modules: diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 6244e3264..cc5f9e487 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -8,11 +8,12 @@ from __future__ import annotations import logging import warnings -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response +from scrapy.utils.python import global_object_name from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider @@ -30,6 +31,21 @@ if TYPE_CHECKING: from scrapy.utils.log import SpiderLoggerAdapter +_SpiderT = TypeVar("_SpiderT", bound="type[Spider]") + + +def ignore_spider(cls: _SpiderT) -> _SpiderT: + """Mark a :class:`~scrapy.spiders.Spider` subclass to be ignored. + + Marked spider classes are not available to the :command:`crawl`, + :command:`list` and :command:`runspider` commands. Only the decorated + class is marked; its subclasses are unaffected. + """ + cls._ignore_spider = True + return cls + + +@ignore_spider class Spider(object_ref): """Base class that any spider must subclass. @@ -40,6 +56,7 @@ class Spider(object_ref): name: str custom_settings: dict[str, Any] | None = None + _ignore_spider: bool #: Start URLs. See :meth:`start`. start_urls: list[str] @@ -48,11 +65,23 @@ class Spider(object_ref): if name is not None: self.name: str = name elif not getattr(self, "name", None): - raise ValueError(f"{type(self).__name__} must have a name") + self.name = type(self)._default_name() self.__dict__.update(kwargs) if not hasattr(self, "start_urls"): self.start_urls: list[str] = [] + @classmethod + def _default_name(cls) -> str: + """Return the name under which spider loaders and commands know this + spider class, which falls back to its import path.""" + return getattr(cls, "name", None) or global_object_name(cls) + + @classmethod + def _is_ignored(cls) -> bool: + # The mark set by ignore_spider() is read from the class __dict__ so + # that subclasses do not inherit it. + return "_ignore_spider" in cls.__dict__ + @property def logger(self) -> SpiderLoggerAdapter: # circular import diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 373c0b8b5..181dfda08 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -16,7 +16,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.link import Link from scrapy.linkextractors import LinkExtractor -from scrapy.spiders import Spider +from scrapy.spiders import Spider, ignore_spider from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.python import global_object_name @@ -95,6 +95,7 @@ class Rule: ) +@ignore_spider class CrawlSpider(Spider): rules: Sequence[Rule] = () _rules: list[Rule] diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 1e7ac9c34..b2f12c990 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotSupported from scrapy.http import Response, TextResponse from scrapy.selector import Selector -from scrapy.spiders import Spider +from scrapy.spiders import Spider, ignore_spider from scrapy.utils.iterators import csviter, xmliter_lxml from scrapy.utils.spider import iterate_spider_output @@ -20,6 +20,7 @@ if TYPE_CHECKING: from collections.abc import Iterable, Sequence +@ignore_spider class XMLFeedSpider(Spider): """ This class intends to be the base class for spiders that scrape @@ -107,6 +108,7 @@ class XMLFeedSpider(Spider): selector.register_namespace(prefix, uri) +@ignore_spider class CSVFeedSpider(Spider): """Spider for parsing CSV feeds. It receives a CSV file in a response; iterates through each of its rows, diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 2a80b8d24..cfbe8d2e3 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -8,7 +8,7 @@ from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse -from scrapy.spiders import Spider +from scrapy.spiders import Spider, ignore_spider from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots @@ -23,6 +23,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +@ignore_spider class SitemapSpider(Spider): sitemap_urls: Sequence[str] = () sitemap_rules: Sequence[tuple[re.Pattern[str] | str, str | CallbackT]] = [ diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 0432a7231..2735c5477 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -83,5 +83,8 @@ DOWNLOAD_DELAY = 1 #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" +# Allow listing and running spiders that do not have a name +SPIDER_LOADER_REQUIRE_NAME = False + # Set settings whose default value is deprecated to a future-proof value FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 9f43df1ce..42b43b7d0 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -47,18 +47,27 @@ def iterate_spider_output( return arg_to_iter(d) -def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]: - """Return an iterator over all spider classes defined in the given module - that can be instantiated (i.e. which have name) +def iter_spider_classes( + module: ModuleType, + *, + require_name: bool = True, +) -> Iterable[type[Spider]]: + """Return an iterator over all :class:`~scrapy.spiders.Spider` subclasses + defined in the given module, excluding those marked with + :func:`scrapy.spiders.ignore_spider`. + + If `require_name` is ``True`` (default), any + :class:`~scrapy.spiders.Spider` subclass without a non-empty + :attr:`~scrapy.Spider.name` is also excluded. """ for obj in vars(module).values(): - if ( - inspect.isclass(obj) - and issubclass(obj, Spider) - and obj.__module__ == module.__name__ - and getattr(obj, "name", None) - ): - yield obj + if not inspect.isclass(obj) or not issubclass(obj, Spider): + continue + if obj.__module__ != module.__name__ or obj._is_ignored(): + continue + if require_name and not getattr(obj, "name", None): + continue + yield obj @overload diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index f67853ece..22956d442 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -28,7 +28,10 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: def _spider_domains(spider: type[Spider]) -> Iterable[str]: - yield spider.name + # Spiders that get their name from their import path have no class-level + # name, and an import path is never a domain anyway. + if name := getattr(spider, "name", None): + yield name if allowed_domains := getattr(spider, "allowed_domains", None): yield from allowed_domains diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 240f44584..a8ccf92c1 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -122,6 +122,25 @@ class CheckSpider(scrapy.Spider): """ self._test_contract(proj_path, contracts, parse_def) + def test_check_list_nameless_spider(self, proj_path: Path) -> None: + spider = proj_path / self.project_name / "spiders" / "namelessspider.py" + spider.write_text( + ''' +import scrapy + +class NamelessSpider(scrapy.Spider): + def parse(self, response): + """ + @url data:, + """ +''', + encoding="utf-8", + ) + name = f"{self.project_name}.spiders.namelessspider.NamelessSpider" + ret, out, err = proc("check", "-l", name, cwd=proj_path) + assert ret == 0, err + assert out == f"{name}\n * parse\n" + def test_SCRAPY_CHECK_set(self, proj_path: Path) -> None: parse_def = """ import os diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index 5306e3bf8..94b1cc21a 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -35,6 +35,24 @@ class TestCrawlCommand(TestProjectBase): "running 'scrapy crawl' with more than one spider is not supported" in err ) + def test_nameless_spider(self, proj_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + async def start(self): + self.logger.debug('It works!') + return + yield +""" + (proj_path / self.project_name / "spiders" / "myspider.py").write_text( + spider_code, encoding="utf-8" + ) + name = f"{self.project_name}.spiders.myspider.MySpider" + _, _, log = proc("crawl", name, cwd=proj_path) + assert f"[{name}] DEBUG: It works!" in log + assert "Spider closed (finished)" in log + def test_no_output(self, proj_path: Path) -> None: spider_code = """ import scrapy diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 11036eaeb..6e8213b67 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -132,6 +132,39 @@ class MySpider(scrapy.Spider): assert ("[scrapy]" in log1) is value assert ("[scrapy.core.engine]" in log1) is not value + def test_runspider_nameless_spider(self, tmp_path: Path) -> None: + nameless_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + async def start(self): + self.logger.debug("It Works!") + return + yield +""" + log = self.get_log(tmp_path, nameless_spider) + assert "[myspider.MySpider] DEBUG: It Works!" in log + assert "INFO: Spider closed (finished)" in log + + def test_runspider_prefers_named_spider(self, tmp_path: Path) -> None: + """A base spider defined after the spider itself does not shadow it.""" + base_last_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug("It Works!") + return + yield + +class MyBaseSpider(scrapy.Spider): + pass +""" + log = self.get_log(tmp_path, base_last_spider) + assert "[myspider] DEBUG: It Works!" in log + def test_runspider_no_spider_found(self, tmp_path: Path) -> None: log = self.get_log(tmp_path, "from scrapy.spiders import Spider\n") assert "No spider found in file" in log diff --git a/tests/test_commands.py b/tests/test_commands.py index f20ecc153..943a9c43b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -432,6 +432,31 @@ class TestMiscCommands(TestProjectBase): subdir.mkdir(exist_ok=True) assert call("list", cwd=subdir) == 0 + @pytest.mark.parametrize("require_name", [False, True]) + def test_list_nameless(self, proj_path: Path, require_name: bool) -> None: + (proj_path / self.project_name / "spiders" / "nameless.py").write_text( + "from scrapy import Spider\n" + "\n" + "\n" + "class NamelessSpider(Spider):\n" + " pass\n" + "\n" + "\n" + "class NamedSpider(Spider):\n" + ' name = "named"\n', + encoding="utf-8", + ) + returncode, out, err = proc( + "list", + "-s", + f"SPIDER_LOADER_REQUIRE_NAME={require_name}", + cwd=proj_path, + ) + assert returncode == 0, err + nameless = f"{self.project_name}.spiders.nameless.NamelessSpider" + expected = ["named"] if require_name else ["named", nameless] + assert out.split() == expected + class TestCommandListing(TestProjectBase): """Tests for the command list that ``scrapy`` prints when called without a diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 81840f7e0..951d4fa76 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,6 +1,7 @@ import contextlib import shutil import sys +import warnings from pathlib import Path from unittest import mock @@ -13,6 +14,7 @@ from scrapy.crawler import CrawlerRunner from scrapy.http import Request from scrapy.settings import Settings from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader +from tests.test_spiderloader.nameless_spiders.nameless1 import NamelessSpider module_dir = Path(__file__).resolve().parent @@ -165,6 +167,69 @@ class TestSpiderLoader: assert not spiders +class TestNamelessSpiderLoader: + module = "tests.test_spiderloader.nameless_spiders" + nameless1 = f"{module}.nameless1.NamelessSpider" + nameless2 = f"{module}.nameless2.NamelessSpider" + + @pytest.fixture + def spider_loader(self): + settings = Settings( + { + "SPIDER_MODULES": [self.module], + "SPIDER_LOADER_REQUIRE_NAME": False, + } + ) + return SpiderLoader.from_settings(settings) + + def test_list(self, spider_loader): + assert set(spider_loader.list()) == { + "subclass", + self.nameless1, + self.nameless2, + } + + def test_list_require_name(self): + settings = Settings({"SPIDER_MODULES": [self.module]}) + spider_loader = SpiderLoader.from_settings(settings) + assert set(spider_loader.list()) == {"subclass"} + + def test_load(self, spider_loader): + assert spider_loader.load(self.nameless1) is NamelessSpider + + def test_instance_name(self, spider_loader): + """Spiders are instantiated with the name that the loader knows them + by.""" + for name in spider_loader.list(): + assert spider_loader.load(name)().name == name + + def test_find_by_request(self, spider_loader): + assert spider_loader.find_by_request( + Request("https://nameless.example.com") + ) == [self.nameless1] + + def test_no_dupename_warning(self): + settings = Settings( + { + "SPIDER_MODULES": [self.module], + "SPIDER_LOADER_REQUIRE_NAME": False, + } + ) + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + SpiderLoader.from_settings(settings) + + def test_crawler_runner_loading(self, spider_loader): + runner = CrawlerRunner( + { + "SPIDER_MODULES": [self.module], + "SPIDER_LOADER_REQUIRE_NAME": False, + } + ) + crawler = runner.create_crawler(self.nameless1) + assert crawler.spidercls is NamelessSpider + + class TestDuplicateSpiderNameLoader: def test_dupename_warning(self, spider_loader_env): settings, spiders_dir = spider_loader_env diff --git a/tests/test_spiderloader/nameless_spiders/__init__.py b/tests/test_spiderloader/nameless_spiders/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_spiderloader/nameless_spiders/ignored.py b/tests/test_spiderloader/nameless_spiders/ignored.py new file mode 100644 index 000000000..fd9113eae --- /dev/null +++ b/tests/test_spiderloader/nameless_spiders/ignored.py @@ -0,0 +1,10 @@ +from scrapy.spiders import Spider, ignore_spider + + +@ignore_spider +class IgnoredSpider(Spider): + name = "ignored" + + +class SubclassSpider(IgnoredSpider): + name = "subclass" diff --git a/tests/test_spiderloader/nameless_spiders/nameless1.py b/tests/test_spiderloader/nameless_spiders/nameless1.py new file mode 100644 index 000000000..dc910cdd4 --- /dev/null +++ b/tests/test_spiderloader/nameless_spiders/nameless1.py @@ -0,0 +1,5 @@ +from scrapy.spiders import Spider + + +class NamelessSpider(Spider): + allowed_domains = ["nameless.example.com"] diff --git a/tests/test_spiderloader/nameless_spiders/nameless2.py b/tests/test_spiderloader/nameless_spiders/nameless2.py new file mode 100644 index 000000000..83c0364d8 --- /dev/null +++ b/tests/test_spiderloader/nameless_spiders/nameless2.py @@ -0,0 +1,7 @@ +from scrapy.spiders import Spider + + +# Same class name as in the nameless1 module, to check that nameless spiders +# are told apart by their full import path. +class NamelessSpider(Spider): + pass diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 71f0f76ad..e1eddf052 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -3,15 +3,46 @@ from __future__ import annotations from scrapy import Spider from scrapy.http import Request from scrapy.item import Item +from scrapy.spiders import ignore_spider from scrapy.utils.spider import iter_spider_classes, iterate_spider_output -class MySpider1(Spider): - name = "myspider1" +class SpiderA(Spider): + pass -class MySpider2(Spider): - name = "myspider2" +@ignore_spider +class SpiderB(Spider): + pass + + +@ignore_spider +class SpiderC(Spider): + name = "c" + + +class SpiderA1(SpiderA): + name = "a1" + + +class SpiderA2(SpiderA): + pass + + +class SpiderB1(SpiderB): + name = "b1" + + +class SpiderB2(SpiderB): + pass + + +class SpiderC1(SpiderC): + name = "c1" + + +class SpiderC2(SpiderC): + pass def test_iterate_spider_output(): @@ -25,8 +56,23 @@ def test_iterate_spider_output(): assert list(iterate_spider_output([r, i, o])) == [r, i, o] -def test_iter_spider_classes(): +def test_iter_spider_classes_require_name(): import tests.test_utils_spider # noqa: PLW0406,PLC0415 - it = iter_spider_classes(tests.test_utils_spider) - assert set(it) == {MySpider1, MySpider2} + it = iter_spider_classes(tests.test_utils_spider, require_name=True) + assert set(it) == {SpiderA1, SpiderB1, SpiderC1, SpiderC2} + + +def test_iter_spider_classes_dont_require_name(): + import tests.test_utils_spider # noqa: PLW0406,PLC0415 + + it = iter_spider_classes(tests.test_utils_spider, require_name=False) + assert set(it) == { + SpiderA, + SpiderA1, + SpiderA2, + SpiderB1, + SpiderB2, + SpiderC1, + SpiderC2, + } diff --git a/tests/utils/bases/spider.py b/tests/utils/bases/spider.py index 799c1820d..db3b0033e 100644 --- a/tests/utils/bases/spider.py +++ b/tests/utils/bases/spider.py @@ -11,6 +11,7 @@ from scrapy import signals from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings +from scrapy.utils.python import global_object_name from scrapy.utils.test import get_crawler, get_reactor_settings from tests.utils.decorators import inline_callbacks_test @@ -35,12 +36,20 @@ class TestSpiderBase(ABC): assert spider.foo == "bar" def test_spider_without_name(self): - """``__init__`` raises when the name is not provided.""" - msg = "must have a name" - with pytest.raises(ValueError, match=msg): - self.spider_class() - with pytest.raises(ValueError, match=msg): - self.spider_class(somearg="foo") + """Spiders with no name get their import path as name.""" + assert not hasattr(self.spider_class, "name") + spider = self.spider_class() + assert spider.name == global_object_name(self.spider_class) + + def test_ignored(self): + """Base spiders shipped by Scrapy are ignored, their subclasses are + not.""" + + class Subclass(self.spider_class): + pass + + assert self.spider_class._is_ignored() + assert not Subclass._is_ignored() def test_from_crawler_crawler_and_settings_population(self): crawler = get_crawler() diff --git a/tox.ini b/tox.ini index 7ea1c57d7..3d150ac86 100644 --- a/tox.ini +++ b/tox.ini @@ -113,7 +113,8 @@ commands = pre-commit run {posargs:--all-files} [testenv:pylint] -# Some checks are Python-version-dependent, so pin the version used in CI. +# Version-dependent code and pylint suppressions require a fixed interpreter. +# Keep in sync with the pylint job in .github/workflows/checks.yml. basepython = python3.14 deps = {[testenv:extra-deps]deps}