diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 0be8ec302..157526f0c 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -312,8 +312,9 @@ list * Syntax: ``scrapy list`` * Requires project: *yes* -List all :ref:`concrete spiders ` available in -the current project. The output is one spider per line. +List all :ref:`spiders ` available in the current project, +excluding :ref:`base spiders `. The output is one spider per +line. Usage example:: @@ -321,6 +322,9 @@ Usage example:: spider1 spider2 +Which spiders are listed depends on the configured +:setting:`SPIDER_LOADER_CLASS`. + .. command:: edit edit diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 6b2fd79cd..47456ae57 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1300,11 +1300,11 @@ Default: ``True`` By default, when loading spiders, Scrapy only loads :class:`~scrapy.spiders.Spider` subclasses that have a :class:`~scrapy.spiders.Spider.name` unless they are decorated with -:func:`~scrapy.spiders.abstractspider`. +:func:`~scrapy.spiders.basespider`. If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``False``, Scrapy loads all Spider subclasses unless they are decorated with -:func:`~scrapy.spiders.abstractspider`. If they do not have a +:func:`~scrapy.spiders.basespider`. If they do not have a :class:`~scrapy.spiders.Spider.name`, their fully-qualified class name is used as a name. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 072c101ba..771464e84 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -61,11 +61,14 @@ scrapy.Spider .. attribute:: name - A string which defines the name for this spider. The spider name is how - the spider is located (and instantiated) by Scrapy, so it must be - unique. However, nothing prevents you from instantiating more than one - instance of the same spider. This is the most important spider attribute - and it's required. + A string which defines the name for this spider. + + If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` (default) and you + use the default Scrapy spider loader (see + :setting:`SPIDER_LOADER_CLASS`), a non-empty name is required for the + spider to be discoverable by Scrapy, and the spider name must be + unique to one spider class; however, nothing prevents you from + instantiating more than one instance of the same spider. If the spider scrapes a single domain, a common practice is to name the spider after the domain, with or without the `TLD`_. So, for example, a @@ -817,40 +820,31 @@ Combine SitemapSpider with other sources of urls:: .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/ -.. _abstract-and-concrete-spiders: +.. _base-spiders: -Abstract and Concrete Spiders -============================= +Base spiders +============ -Abstract spiders are :class:`~scrapy.spiders.Spider` subclasses that are -not loaded by the default spider loader (see :setting:`SPIDER_LOADER_CLASS`). -Abstract spiders cannot be executed, they can only be subclassed to create -other spiders. +Base spiders are :class:`~scrapy.spiders.Spider` subclasses that are not meant +to be run by Scrapy. They are only meant to be subclassed to create regular +spiders. They are one way to share code between two or more spiders. -To be able to use a spider, you must mark it as a concrete spider. +Use the :func:`~scrapy.spiders.basespider` decorator to mark a spider class as +a base spider, so that the default Scrapy spider loader (see +:setting:`SPIDER_LOADER_CLASS`) ignores that spider class, hence preventing +Scrapy from running or listing (see the :command:`list` command) that spider +class. -How you mark a spider as a concrete spider depends on the value of the -:setting:`SPIDER_LOADER_REQUIRE_NAME` setting: - -- If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` (default), add a - non-empty :class:`~scrapy.spiders.Spider.name` to a spider to make it a - concrete spider. - -- If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``False``, all spiders are - considered concrete spiders by default. - -Use :func:`~scrapy.spiders.abstractspider` to mark a spider as an abstract -spider: - -.. autodecorator:: scrapy.spiders.abstractspider +.. autodecorator:: scrapy.spiders.basespider For example:: - from scrapy.spiders import abstractspider, Spider + from scrapy.spiders import basespider, Spider - @abstractspider + @basespider class MyBaseSpider(Spider): pass - class MySpider(MyBaseSpider): - pass +If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` (default), any +:class:`~scrapy.spiders.Spider` subclass without a ``name`` class attribute is +also treated as a base spider. diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 62b7116be..8641ea729 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -13,9 +13,9 @@ from scrapy.utils.url import url_is_from_spider from scrapy.utils.deprecate import method_is_overridden -def abstractspider(decorated_cls): - """Marks a :class:`~scrapy.spiders.Spider` subclass as an :ref:`abstract - spider `.""" +def basespider(decorated_cls): + """Marks a :class:`~scrapy.spiders.Spider` subclass as a :ref:`base spider + `.""" @classmethod def is_abstract(cls): diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 07984cb5e..c194f84e2 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -11,7 +11,7 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor -from scrapy.spiders import abstractspider, Spider +from scrapy.spiders import basespider, Spider from scrapy.utils.python import get_func_args from scrapy.utils.spider import iterate_spider_output @@ -66,7 +66,7 @@ class Rule: return self.process_request(*args) -@abstractspider +@basespider class CrawlSpider(Spider): rules = () diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 2fba8f1bf..bcd1cff32 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -4,14 +4,14 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ -from scrapy.spiders import abstractspider, Spider +from scrapy.spiders import basespider, Spider from scrapy.utils.iterators import xmliter, csviter from scrapy.utils.spider import iterate_spider_output from scrapy.selector import Selector from scrapy.exceptions import NotConfigured, NotSupported -@abstractspider +@basespider class XMLFeedSpider(Spider): """ This class intends to be the base class for spiders that scrape @@ -92,7 +92,7 @@ class XMLFeedSpider(Spider): selector.register_namespace(prefix, uri) -@abstractspider +@basespider class CSVFeedSpider(Spider): """Spider for parsing CSV feeds. It receives a CSV file in a response; iterates through each of its rows, diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index f55503137..92dae52b3 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,8 +1,8 @@ -from scrapy.spiders import abstractspider, Spider +from scrapy.spiders import basespider, Spider from scrapy.utils.spider import iterate_spider_output -@abstractspider +@basespider class InitSpider(Spider): """Base Spider with initialization facilities""" diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 888a39ba3..65947dbe8 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,7 +1,7 @@ import re import logging -from scrapy.spiders import abstractspider, Spider +from scrapy.spiders import basespider, Spider from scrapy.http import Request, XmlResponse from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots from scrapy.utils.gz import gunzip, gzip_magic_number @@ -10,7 +10,7 @@ from scrapy.utils.gz import gunzip, gzip_magic_number logger = logging.getLogger(__name__) -@abstractspider +@basespider class SitemapSpider(Spider): sitemap_urls = () diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index a1c30190d..4f038044b 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -21,7 +21,7 @@ def iterate_spider_output(result): return arg_to_iter(deferred_from_coro(result)) -def _is_concrete_spider(spider_class, require_name): +def _is_non_base_spider(spider_class, require_name): return ( inspect.isclass(spider_class) and issubclass(spider_class, Spider) @@ -34,20 +34,20 @@ def _is_concrete_spider(spider_class, require_name): def iter_spider_classes(module, *, require_name=True): - """Return an iterator over all :ref:`concrete spider - ` classes defined in the given module. + """Return an iterator over all :ref:`spider ` classes + defined in the given module, excluding :ref:`base spiders `. If `require_name` is ``True`` (default), any :class:`~scrapy.spiders.Spider` subclass with a non-empty :class:`~scrapy.spiders.Spider.name` and not decorated with - :func:`~scrapy.spiders.abstractspider` is considered a concrete spider. + :func:`~scrapy.spiders.basespider` is yielded. If `require_name` is ``False``, any :class:`~scrapy.spiders.Spider` - subclass not decorated with :func:`~scrapy.spiders.abstractspider` is - considered a concrete spider. + subclass not decorated with :func:`~scrapy.spiders.basespider` is + yielded. """ for obj in vars(module).values(): - if (_is_concrete_spider(obj, require_name) + if (_is_non_base_spider(obj, require_name) and obj.__module__ == module.__name__): yield obj diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index f485c2299..aa8128c4e 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -3,7 +3,7 @@ import unittest from scrapy import Spider from scrapy.http import Request from scrapy.item import BaseItem -from scrapy.spiders import abstractspider +from scrapy.spiders import basespider from scrapy.utils.spider import iterate_spider_output, iter_spider_classes @@ -11,12 +11,12 @@ class SpiderA(Spider): pass -@abstractspider +@basespider class SpiderB(Spider): pass -@abstractspider +@basespider class SpiderC(Spider): name = 'c'