abstractspider → basespider

docs/topics/commands.rst

@@ -312,8 +312,9 @@ list
 * Syntax: ``scrapy list``
 * Requires project: *yes*
 
-List all :ref:`concrete spiders <abstract-and-concrete-spiders>` available in
-the current project. The output is one spider per line.
+List all :ref:`spiders <topics-spiders>` available in the current project,
+excluding :ref:`base spiders <base-spiders>`. The output is one spider per
+line.
 
 Usage example::
 
@@ -321,6 +322,9 @@ Usage example::
     spider1
     spider2
 
+Which spiders are listed depends on the configured
+:setting:`SPIDER_LOADER_CLASS`.
+
 .. command:: edit
 
 edit

docs/topics/settings.rst

@@ -1300,11 +1300,11 @@ Default: ``True``
 By default, when loading spiders, Scrapy only loads
 :class:`~scrapy.spiders.Spider` subclasses that have a
 :class:`~scrapy.spiders.Spider.name` unless they are decorated with
-:func:`~scrapy.spiders.abstractspider`.
+:func:`~scrapy.spiders.basespider`.
 
 If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``False``, Scrapy loads all Spider
 subclasses unless they are decorated with
-:func:`~scrapy.spiders.abstractspider`. If they do not have a
+:func:`~scrapy.spiders.basespider`. If they do not have a
 :class:`~scrapy.spiders.Spider.name`, their fully-qualified class name is used
 as a name.
 

docs/topics/spiders.rst

@@ -61,11 +61,14 @@ scrapy.Spider
 
    .. attribute:: name
 
-       A string which defines the name for this spider. The spider name is how
-       the spider is located (and instantiated) by Scrapy, so it must be
-       unique. However, nothing prevents you from instantiating more than one
-       instance of the same spider. This is the most important spider attribute
-       and it's required.
+       A string which defines the name for this spider.
+
+       If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` (default) and you
+       use the default Scrapy spider loader (see
+       :setting:`SPIDER_LOADER_CLASS`), a non-empty name is required for the
+       spider to be discoverable by Scrapy, and the spider name must be
+       unique to one spider class; however, nothing prevents you from
+       instantiating more than one instance of the same spider.
 
        If the spider scrapes a single domain, a common practice is to name the
        spider after the domain, with or without the `TLD`_. So, for example, a
@@ -817,40 +820,31 @@ Combine SitemapSpider with other sources of urls::
 .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/
 
 
-.. _abstract-and-concrete-spiders:
+.. _base-spiders:
 
-Abstract and Concrete Spiders
-=============================
+Base spiders
+============
 
-Abstract spiders are :class:`~scrapy.spiders.Spider` subclasses that are
-not loaded by the default spider loader (see :setting:`SPIDER_LOADER_CLASS`).
-Abstract spiders cannot be executed, they can only be subclassed to create
-other spiders.
+Base spiders are :class:`~scrapy.spiders.Spider` subclasses that are not meant
+to be run by Scrapy. They are only meant to be subclassed to create regular
+spiders. They are one way to share code between two or more spiders.
 
-To be able to use a spider, you must mark it as a concrete spider.
+Use the :func:`~scrapy.spiders.basespider` decorator to mark a spider class as
+a base spider, so that the default Scrapy spider loader (see
+:setting:`SPIDER_LOADER_CLASS`) ignores that spider class, hence preventing
+Scrapy from running or listing (see the :command:`list` command) that spider
+class.
 
-How you mark a spider as a concrete spider depends on the value of the
-:setting:`SPIDER_LOADER_REQUIRE_NAME` setting:
-
--   If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` (default), add a
-    non-empty :class:`~scrapy.spiders.Spider.name` to a spider to make it a
-    concrete spider.
-
--   If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``False``, all spiders are
-    considered concrete spiders by default.
-
-Use :func:`~scrapy.spiders.abstractspider` to mark a spider as an abstract
-spider:
-
-.. autodecorator:: scrapy.spiders.abstractspider
+.. autodecorator:: scrapy.spiders.basespider
 
 For example::
 
-    from scrapy.spiders import abstractspider, Spider
+    from scrapy.spiders import basespider, Spider
 
-    @abstractspider
+    @basespider
     class MyBaseSpider(Spider):
         pass
 
-    class MySpider(MyBaseSpider):
-        pass
+If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` (default), any
+:class:`~scrapy.spiders.Spider` subclass without a ``name`` class attribute is
+also treated as a base spider.

scrapy/spiders/__init__.py

@@ -13,9 +13,9 @@ from scrapy.utils.url import url_is_from_spider
 from scrapy.utils.deprecate import method_is_overridden
 
 
-def abstractspider(decorated_cls):
-    """Marks a :class:`~scrapy.spiders.Spider` subclass as an :ref:`abstract
-    spider <abstract-and-concrete-spiders>`."""
+def basespider(decorated_cls):
+    """Marks a :class:`~scrapy.spiders.Spider` subclass as a :ref:`base spider
+    <base-spiders>`."""
 
     @classmethod
     def is_abstract(cls):

scrapy/spiders/crawl.py

@@ -11,7 +11,7 @@ import warnings
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
-from scrapy.spiders import abstractspider, Spider
+from scrapy.spiders import basespider, Spider
 from scrapy.utils.python import get_func_args
 from scrapy.utils.spider import iterate_spider_output
 
@@ -66,7 +66,7 @@ class Rule:
         return self.process_request(*args)
 
 
-@abstractspider
+@basespider
 class CrawlSpider(Spider):
 
     rules = ()

scrapy/spiders/feed.py

@@ -4,14 +4,14 @@ for scraping from an XML feed.
 
 See documentation in docs/topics/spiders.rst
 """
-from scrapy.spiders import abstractspider, Spider
+from scrapy.spiders import basespider, Spider
 from scrapy.utils.iterators import xmliter, csviter
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.selector import Selector
 from scrapy.exceptions import NotConfigured, NotSupported
 
 
-@abstractspider
+@basespider
 class XMLFeedSpider(Spider):
     """
     This class intends to be the base class for spiders that scrape
@@ -92,7 +92,7 @@ class XMLFeedSpider(Spider):
             selector.register_namespace(prefix, uri)
 
 
-@abstractspider
+@basespider
 class CSVFeedSpider(Spider):
     """Spider for parsing CSV feeds.
     It receives a CSV file in a response; iterates through each of its rows,

scrapy/spiders/init.py

@@ -1,8 +1,8 @@
-from scrapy.spiders import abstractspider, Spider
+from scrapy.spiders import basespider, Spider
 from scrapy.utils.spider import iterate_spider_output
 
 
-@abstractspider
+@basespider
 class InitSpider(Spider):
     """Base Spider with initialization facilities"""
 

scrapy/spiders/sitemap.py

@@ -1,7 +1,7 @@
 import re
 import logging
 
-from scrapy.spiders import abstractspider, Spider
+from scrapy.spiders import basespider, Spider
 from scrapy.http import Request, XmlResponse
 from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
 from scrapy.utils.gz import gunzip, gzip_magic_number
@@ -10,7 +10,7 @@ from scrapy.utils.gz import gunzip, gzip_magic_number
 logger = logging.getLogger(__name__)
 
 
-@abstractspider
+@basespider
 class SitemapSpider(Spider):
 
     sitemap_urls = ()

scrapy/utils/spider.py

@@ -21,7 +21,7 @@ def iterate_spider_output(result):
     return arg_to_iter(deferred_from_coro(result))
 
 
-def _is_concrete_spider(spider_class, require_name):
+def _is_non_base_spider(spider_class, require_name):
     return (
         inspect.isclass(spider_class)
         and issubclass(spider_class, Spider)
@@ -34,20 +34,20 @@ def _is_concrete_spider(spider_class, require_name):
 
 
 def iter_spider_classes(module, *, require_name=True):
-    """Return an iterator over all :ref:`concrete spider
-    <abstract-and-concrete-spiders>` classes defined in the given module.
+    """Return an iterator over all :ref:`spider <topics-spiders>` classes
+    defined in the given module, excluding :ref:`base spiders <base-spiders>`.
 
     If `require_name` is ``True`` (default), any
     :class:`~scrapy.spiders.Spider` subclass with a non-empty
     :class:`~scrapy.spiders.Spider.name` and not decorated with
-    :func:`~scrapy.spiders.abstractspider` is considered a concrete spider.
+    :func:`~scrapy.spiders.basespider` is yielded.
 
     If `require_name` is ``False``, any :class:`~scrapy.spiders.Spider`
-    subclass not decorated with :func:`~scrapy.spiders.abstractspider` is
-    considered a concrete spider.
+    subclass not decorated with :func:`~scrapy.spiders.basespider` is
+    yielded.
     """
     for obj in vars(module).values():
-        if (_is_concrete_spider(obj, require_name)
+        if (_is_non_base_spider(obj, require_name)
                 and obj.__module__ == module.__name__):
             yield obj
 

tests/test_utils_spider.py

@@ -3,7 +3,7 @@ import unittest
 from scrapy import Spider
 from scrapy.http import Request
 from scrapy.item import BaseItem
-from scrapy.spiders import abstractspider
+from scrapy.spiders import basespider
 from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
 
 
@@ -11,12 +11,12 @@ class SpiderA(Spider):
     pass
 
 
-@abstractspider
+@basespider
 class SpiderB(Spider):
     pass
 
 
-@abstractspider
+@basespider
 class SpiderC(Spider):
     name = 'c'