@basespider → @ignore_spider

docs/topics/commands.rst

@@ -310,9 +310,8 @@ list
 * Syntax: ``scrapy list``
 * Requires project: *yes*
 
-List all :ref:`spiders <topics-spiders>` available in the current project,
-excluding :ref:`base spiders <base-spiders>`. The output is one spider per
-line.
+List all :ref:`spiders <topics-spiders>` available in the current project. The
+output is one spider per line.
 
 Usage example::
 

docs/topics/settings.rst

@@ -1359,13 +1359,13 @@ SPIDER_LOADER_REQUIRE_NAME
 Default: ``True``
 
 By default, when loading spiders, Scrapy only loads
-:class:`~scrapy.spiders.Spider` subclasses that have a
+:class:`~scrapy.spiders.Spider` subclasses that have a non-empty
 :class:`~scrapy.spiders.Spider.name` unless they are decorated with
-:func:`~scrapy.spiders.basespider`.
+:func:`~scrapy.spiders.ignore_spider`.
 
-If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``False``, Scrapy loads all Spider
-subclasses unless they are decorated with
-:func:`~scrapy.spiders.basespider`. If they do not have a
+If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``False``, Scrapy loads all
+:class:`~scrapy.spiders.Spider` subclasses unless they are decorated with
+:func:`~scrapy.spiders.ignore_spider`. If they do not have a non-empty
 :class:`~scrapy.spiders.Spider.name`, their fully-qualified class name is used
 as a name.
 

docs/topics/spiders.rst

@@ -66,9 +66,12 @@ scrapy.Spider
        If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` (default) and you
        use the default Scrapy spider loader (see
        :setting:`SPIDER_LOADER_CLASS`), a non-empty name is required for the
-       spider to be discoverable by Scrapy, and the spider name must be
-       unique to one spider class; however, nothing prevents you from
-       instantiating more than one instance of the same spider.
+       spider to be discoverable by the Scrapy commands :command:`crawl`,
+       :command:`list`, and :command:`runspider`.
+
+       The spider name must be unique to one spider class. If two or more
+       spiders have the same name, Scrapy commands :command:`crawl` and
+       :command:`runspider` will only be able to run one of the spiders.
 
        If the spider scrapes a single domain, a common practice is to name the
        spider after the domain, with or without the `TLD`_. So, for example, a
@@ -831,31 +834,27 @@ Combine SitemapSpider with other sources of urls::
 .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/
 
 
-.. _base-spiders:
-
 Base spiders
 ============
 
 Base spiders are :class:`~scrapy.spiders.Spider` subclasses that are not meant
 to be run by Scrapy. They are only meant to be subclassed to create regular
-spiders. They are one way to share code between two or more spiders.
+spiders or other base spiders. They are one way to share code between two or
+more spiders.
 
-Use the :func:`~scrapy.spiders.basespider` decorator to mark a spider class as
-a base spider, so that the default Scrapy spider loader (see
-:setting:`SPIDER_LOADER_CLASS`) ignores that spider class, hence preventing
-Scrapy from running or listing (see the :command:`list` command) that spider
-class.
+Use the :func:`~scrapy.spiders.ignore_spider` decorator to mark any base spider
+class:
 
-.. autodecorator:: scrapy.spiders.basespider
+.. autodecorator:: scrapy.spiders.ignore_spider
 
 For example::
 
-    from scrapy.spiders import basespider, Spider
+    from scrapy.spiders import ignore_spider, Spider
 
-    @basespider
+    @ignore_spider
     class MyBaseSpider(Spider):
         pass
 
 If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` (default), any
 :class:`~scrapy.spiders.Spider` subclass without a ``name`` class attribute is
-also treated as a base spider.
+also ignored.

scrapy/commands/runspider.py

@@ -48,8 +48,7 @@ class Command(BaseRunSpiderCommand):
         except (ImportError, ValueError) as e:
             raise UsageError(f"Unable to load {filename!r}: {e}\n")
         require_name = self.settings.getbool('SPIDER_LOADER_REQUIRE_NAME')
-        spclasses = list(iter_spider_classes(module,
-                                             require_name=require_name))
+        spclasses = list(iter_spider_classes(module, require_name=require_name))
         if not spclasses:
             raise UsageError(f"No spider found in file: {filename}\n")
         spidercls = spclasses.pop()

scrapy/spiders/__init__.py

@@ -14,17 +14,21 @@ from scrapy.utils.url import url_is_from_spider
 from scrapy.utils.deprecate import method_is_overridden
 
 
-def basespider(decorated_cls):
-    """Marks a :class:`~scrapy.spiders.Spider` subclass as a :ref:`base spider
-    <base-spiders>`."""
+def ignore_spider(decorated_cls):
+    """Mark a :class:`~scrapy.spiders.Spider` subclass to be ignored.
+
+    The default spider loader (see :setting:`SPIDER_LOADER_CLASS`) does not
+    make marked spider classes available for the :command:`crawl`,
+    :command:`list`, and :command:`runspider` commands.
+    """
 
     @classmethod
-    def is_abstract(cls):
+    def _is_ignored(cls):
         if cls is decorated_cls:
             return True
-        return super(decorated_cls, cls).is_abstract()
+        return super(decorated_cls, cls)._is_ignored()
 
-    decorated_cls.is_abstract = is_abstract
+    decorated_cls._is_ignored = _is_ignored
     return decorated_cls
 
 
@@ -44,7 +48,7 @@ class Spider(object_ref):
             self.start_urls = []
 
     @classmethod
-    def is_abstract(cls):
+    def _is_ignored(cls):
         return cls is Spider
 
     @property

scrapy/spiders/crawl.py

@@ -10,7 +10,7 @@ from typing import Sequence
 
 from scrapy.http import Request, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
-from scrapy.spiders import basespider, Spider
+from scrapy.spiders import ignore_spider, Spider
 from scrapy.utils.spider import iterate_spider_output
 
 
@@ -59,7 +59,7 @@ class Rule:
         self.process_request = _get_method(self.process_request, spider)
 
 
-@basespider
+@ignore_spider
 class CrawlSpider(Spider):
 
     rules: Sequence[Rule] = ()

scrapy/spiders/feed.py

@@ -4,14 +4,14 @@ for scraping from an XML feed.
 
 See documentation in docs/topics/spiders.rst
 """
-from scrapy.spiders import basespider, Spider
+from scrapy.spiders import ignore_spider, Spider
 from scrapy.utils.iterators import xmliter, csviter
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.selector import Selector
 from scrapy.exceptions import NotConfigured, NotSupported
 
 
-@basespider
+@ignore_spider
 class XMLFeedSpider(Spider):
     """
     This class intends to be the base class for spiders that scrape
@@ -92,7 +92,7 @@ class XMLFeedSpider(Spider):
             selector.register_namespace(prefix, uri)
 
 
-@basespider
+@ignore_spider
 class CSVFeedSpider(Spider):
     """Spider for parsing CSV feeds.
     It receives a CSV file in a response; iterates through each of its rows,

scrapy/spiders/init.py

@@ -1,8 +1,8 @@
-from scrapy.spiders import basespider, Spider
+from scrapy.spiders import ignore_spider, Spider
 from scrapy.utils.spider import iterate_spider_output
 
 
-@basespider
+@ignore_spider
 class InitSpider(Spider):
     """Base Spider with initialization facilities"""
 

scrapy/spiders/sitemap.py

@@ -1,7 +1,7 @@
 import re
 import logging
 
-from scrapy.spiders import basespider, Spider
+from scrapy.spiders import ignore_spider, Spider
 from scrapy.http import Request, XmlResponse
 from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
 from scrapy.utils.gz import gunzip, gzip_magic_number
@@ -10,7 +10,7 @@ from scrapy.utils.gz import gunzip, gzip_magic_number
 logger = logging.getLogger(__name__)
 
 
-@basespider
+@ignore_spider
 class SitemapSpider(Spider):
 
     sitemap_urls = ()

scrapy/utils/spider.py

@@ -25,33 +25,29 @@ def iterate_spider_output(result):
     return arg_to_iter(result)
 
 
-def _is_non_base_spider(spider_class, require_name):
+def _is_ignored(spider_class, *, require_name):
     return (
-        inspect.isclass(spider_class)
-        and issubclass(spider_class, Spider)
-        and not spider_class.is_abstract()
-        and (
-            getattr(spider_class, 'name', None)
-            or not require_name
-        )
+        not inspect.isclass(spider_class)
+        or not issubclass(spider_class, Spider)
+        or spider_class._is_ignored()
+        or require_name and not getattr(spider_class, 'name', None)
     )
 
 
 def iter_spider_classes(module, *, require_name=True):
-    """Return an iterator over all :ref:`spider <topics-spiders>` classes
-    defined in the given module, excluding :ref:`base spiders <base-spiders>`.
+    """Return an iterator over all :class:`~scrapy.spiders.Spider` subclasses
+    defined in the given module, excluding those marked with
+    :func:`scrapy.spiders.ignore_spider`.
 
     If `require_name` is ``True`` (default), any
-    :class:`~scrapy.spiders.Spider` subclass with a non-empty
-    :class:`~scrapy.spiders.Spider.name` and not decorated with
-    :func:`~scrapy.spiders.basespider` is yielded.
-
-    If `require_name` is ``False``, any :class:`~scrapy.spiders.Spider`
-    subclass not decorated with :func:`~scrapy.spiders.basespider` is
-    yielded.
+    :class:`~scrapy.spiders.Spider` subclass without a non-empty
+    :class:`~scrapy.spiders.Spider.name` is also excluded.
     """
     for obj in vars(module).values():
-        if _is_non_base_spider(obj, require_name) and obj.__module__ == module.__name__:
+        if (
+            not _is_ignored(obj, require_name=require_name)
+            and obj.__module__ == module.__name__
+        ):
             yield obj
 
 

tests/test_utils_spider.py

@@ -2,7 +2,7 @@ import unittest
 
 from scrapy import Spider
 from scrapy.http import Request
-from scrapy.spiders import basespider
+from scrapy.spiders import ignore_spider
 from scrapy.item import Item
 from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
 
@@ -11,12 +11,12 @@ class SpiderA(Spider):
     pass
 
 
-@basespider
+@ignore_spider
 class SpiderB(Spider):
     pass
 
 
-@basespider
+@ignore_spider
 class SpiderC(Spider):
     name = 'c'