diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 3c2763917..311c0661b 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -310,9 +310,8 @@ list * Syntax: ``scrapy list`` * Requires project: *yes* -List all :ref:`spiders ` available in the current project, -excluding :ref:`base spiders `. The output is one spider per -line. +List all :ref:`spiders ` available in the current project. The +output is one spider per line. Usage example:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8480381c9..b7e568c72 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1359,13 +1359,13 @@ SPIDER_LOADER_REQUIRE_NAME Default: ``True`` By default, when loading spiders, Scrapy only loads -:class:`~scrapy.spiders.Spider` subclasses that have a +:class:`~scrapy.spiders.Spider` subclasses that have a non-empty :class:`~scrapy.spiders.Spider.name` unless they are decorated with -:func:`~scrapy.spiders.basespider`. +:func:`~scrapy.spiders.ignore_spider`. -If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``False``, Scrapy loads all Spider -subclasses unless they are decorated with -:func:`~scrapy.spiders.basespider`. If they do not have a +If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``False``, Scrapy loads all +:class:`~scrapy.spiders.Spider` subclasses unless they are decorated with +:func:`~scrapy.spiders.ignore_spider`. If they do not have a non-empty :class:`~scrapy.spiders.Spider.name`, their fully-qualified class name is used as a name. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index c640e1d11..1bedea7cb 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -66,9 +66,12 @@ scrapy.Spider If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` (default) and you use the default Scrapy spider loader (see :setting:`SPIDER_LOADER_CLASS`), a non-empty name is required for the - spider to be discoverable by Scrapy, and the spider name must be - unique to one spider class; however, nothing prevents you from - instantiating more than one instance of the same spider. + spider to be discoverable by the Scrapy commands :command:`crawl`, + :command:`list`, and :command:`runspider`. + + The spider name must be unique to one spider class. If two or more + spiders have the same name, Scrapy commands :command:`crawl` and + :command:`runspider` will only be able to run one of the spiders. If the spider scrapes a single domain, a common practice is to name the spider after the domain, with or without the `TLD`_. So, for example, a @@ -831,31 +834,27 @@ Combine SitemapSpider with other sources of urls:: .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/ -.. _base-spiders: - Base spiders ============ Base spiders are :class:`~scrapy.spiders.Spider` subclasses that are not meant to be run by Scrapy. They are only meant to be subclassed to create regular -spiders. They are one way to share code between two or more spiders. +spiders or other base spiders. They are one way to share code between two or +more spiders. -Use the :func:`~scrapy.spiders.basespider` decorator to mark a spider class as -a base spider, so that the default Scrapy spider loader (see -:setting:`SPIDER_LOADER_CLASS`) ignores that spider class, hence preventing -Scrapy from running or listing (see the :command:`list` command) that spider -class. +Use the :func:`~scrapy.spiders.ignore_spider` decorator to mark any base spider +class: -.. autodecorator:: scrapy.spiders.basespider +.. autodecorator:: scrapy.spiders.ignore_spider For example:: - from scrapy.spiders import basespider, Spider + from scrapy.spiders import ignore_spider, Spider - @basespider + @ignore_spider class MyBaseSpider(Spider): pass If :setting:`SPIDER_LOADER_REQUIRE_NAME` is ``True`` (default), any :class:`~scrapy.spiders.Spider` subclass without a ``name`` class attribute is -also treated as a base spider. +also ignored. diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 079a7792c..1034a8d7c 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -48,8 +48,7 @@ class Command(BaseRunSpiderCommand): except (ImportError, ValueError) as e: raise UsageError(f"Unable to load {filename!r}: {e}\n") require_name = self.settings.getbool('SPIDER_LOADER_REQUIRE_NAME') - spclasses = list(iter_spider_classes(module, - require_name=require_name)) + spclasses = list(iter_spider_classes(module, require_name=require_name)) if not spclasses: raise UsageError(f"No spider found in file: {filename}\n") spidercls = spclasses.pop() diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 30af09124..efe8bec13 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -14,17 +14,21 @@ from scrapy.utils.url import url_is_from_spider from scrapy.utils.deprecate import method_is_overridden -def basespider(decorated_cls): - """Marks a :class:`~scrapy.spiders.Spider` subclass as a :ref:`base spider - `.""" +def ignore_spider(decorated_cls): + """Mark a :class:`~scrapy.spiders.Spider` subclass to be ignored. + + The default spider loader (see :setting:`SPIDER_LOADER_CLASS`) does not + make marked spider classes available for the :command:`crawl`, + :command:`list`, and :command:`runspider` commands. + """ @classmethod - def is_abstract(cls): + def _is_ignored(cls): if cls is decorated_cls: return True - return super(decorated_cls, cls).is_abstract() + return super(decorated_cls, cls)._is_ignored() - decorated_cls.is_abstract = is_abstract + decorated_cls._is_ignored = _is_ignored return decorated_cls @@ -44,7 +48,7 @@ class Spider(object_ref): self.start_urls = [] @classmethod - def is_abstract(cls): + def _is_ignored(cls): return cls is Spider @property diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 3709c585e..64f9ecb9d 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -10,7 +10,7 @@ from typing import Sequence from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor -from scrapy.spiders import basespider, Spider +from scrapy.spiders import ignore_spider, Spider from scrapy.utils.spider import iterate_spider_output @@ -59,7 +59,7 @@ class Rule: self.process_request = _get_method(self.process_request, spider) -@basespider +@ignore_spider class CrawlSpider(Spider): rules: Sequence[Rule] = () diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 6e0812404..2ff35045a 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -4,14 +4,14 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ -from scrapy.spiders import basespider, Spider +from scrapy.spiders import ignore_spider, Spider from scrapy.utils.iterators import xmliter, csviter from scrapy.utils.spider import iterate_spider_output from scrapy.selector import Selector from scrapy.exceptions import NotConfigured, NotSupported -@basespider +@ignore_spider class XMLFeedSpider(Spider): """ This class intends to be the base class for spiders that scrape @@ -92,7 +92,7 @@ class XMLFeedSpider(Spider): selector.register_namespace(prefix, uri) -@basespider +@ignore_spider class CSVFeedSpider(Spider): """Spider for parsing CSV feeds. It receives a CSV file in a response; iterates through each of its rows, diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 396ca5184..224d1de90 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,8 +1,8 @@ -from scrapy.spiders import basespider, Spider +from scrapy.spiders import ignore_spider, Spider from scrapy.utils.spider import iterate_spider_output -@basespider +@ignore_spider class InitSpider(Spider): """Base Spider with initialization facilities""" diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 678651e43..c7935e083 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,7 +1,7 @@ import re import logging -from scrapy.spiders import basespider, Spider +from scrapy.spiders import ignore_spider, Spider from scrapy.http import Request, XmlResponse from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots from scrapy.utils.gz import gunzip, gzip_magic_number @@ -10,7 +10,7 @@ from scrapy.utils.gz import gunzip, gzip_magic_number logger = logging.getLogger(__name__) -@basespider +@ignore_spider class SitemapSpider(Spider): sitemap_urls = () diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 6576f9959..2b13f8617 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -25,33 +25,29 @@ def iterate_spider_output(result): return arg_to_iter(result) -def _is_non_base_spider(spider_class, require_name): +def _is_ignored(spider_class, *, require_name): return ( - inspect.isclass(spider_class) - and issubclass(spider_class, Spider) - and not spider_class.is_abstract() - and ( - getattr(spider_class, 'name', None) - or not require_name - ) + not inspect.isclass(spider_class) + or not issubclass(spider_class, Spider) + or spider_class._is_ignored() + or require_name and not getattr(spider_class, 'name', None) ) def iter_spider_classes(module, *, require_name=True): - """Return an iterator over all :ref:`spider ` classes - defined in the given module, excluding :ref:`base spiders `. + """Return an iterator over all :class:`~scrapy.spiders.Spider` subclasses + defined in the given module, excluding those marked with + :func:`scrapy.spiders.ignore_spider`. If `require_name` is ``True`` (default), any - :class:`~scrapy.spiders.Spider` subclass with a non-empty - :class:`~scrapy.spiders.Spider.name` and not decorated with - :func:`~scrapy.spiders.basespider` is yielded. - - If `require_name` is ``False``, any :class:`~scrapy.spiders.Spider` - subclass not decorated with :func:`~scrapy.spiders.basespider` is - yielded. + :class:`~scrapy.spiders.Spider` subclass without a non-empty + :class:`~scrapy.spiders.Spider.name` is also excluded. """ for obj in vars(module).values(): - if _is_non_base_spider(obj, require_name) and obj.__module__ == module.__name__: + if ( + not _is_ignored(obj, require_name=require_name) + and obj.__module__ == module.__name__ + ): yield obj diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 22304016b..c2d3ef099 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -2,7 +2,7 @@ import unittest from scrapy import Spider from scrapy.http import Request -from scrapy.spiders import basespider +from scrapy.spiders import ignore_spider from scrapy.item import Item from scrapy.utils.spider import iterate_spider_output, iter_spider_classes @@ -11,12 +11,12 @@ class SpiderA(Spider): pass -@basespider +@ignore_spider class SpiderB(Spider): pass -@basespider +@ignore_spider class SpiderC(Spider): name = 'c'