From 7a67d768d6b6d82b3858c885f7148c0bb1c9afea Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 29 Jul 2026 15:55:51 +0200 Subject: [PATCH] Use autodoc for spiders and review their docs --- docs/intro/tutorial.rst | 8 +- docs/topics/spiders.rst | 482 ++++--------------------------------- scrapy/spiders/__init__.py | 140 ++++++++++- scrapy/spiders/crawl.py | 79 ++++++ scrapy/spiders/feed.py | 138 ++++++++--- scrapy/spiders/sitemap.py | 106 +++++++- 6 files changed, 473 insertions(+), 480 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index c4e04364b..31923f849 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -762,10 +762,10 @@ much because of a programming mistake. This can be configured in the Hopefully by now you have a good understanding of how to use the mechanism of following links and callbacks with Scrapy. -As yet another example spider that leverages the mechanism of following links, -check out the :class:`~scrapy.spiders.CrawlSpider` class for a generic -spider that implements a small rules engine that you can use to write your -crawlers on top of it. +If the link following of a crawl can be expressed as a set of declarative +rules, the :class:`~scrapy.spiders.CrawlSpider` class implements a small rules +engine that handles it for you. For anything else, keep following links from +your callbacks, as shown above. Also, a common pattern is to build an item with data from more than one page, using a :ref:`trick to pass additional data to the callbacks diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 69ff08fa1..088b81642 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -50,168 +50,48 @@ scrapy.Spider .. class:: scrapy.spiders.Spider .. autoclass:: scrapy.Spider - .. attribute:: name + .. autoattribute:: name - A string which defines the name for this spider. The spider name is how - the spider is located (and instantiated) by Scrapy, so it must be - unique. However, nothing prevents you from instantiating more than one - instance of the same spider. This is the most important spider attribute - and it's required. + .. attribute:: allowed_domains + :type: list[str] - If the spider scrapes a single domain, a common practice is to name the - spider after the domain, with or without the `TLD`_. So, for example, a - spider that crawls ``mywebsite.com`` would often be called - ``mywebsite``. + The domains that this spider is allowed to crawl, if any. Requests for + URLs not belonging to the domain names specified in this list (or their + subdomains) won't be followed if + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is + enabled. - .. attribute:: allowed_domains + Let's say your target url is ``https://www.example.com/1.html``, + then add ``'example.com'`` to the list. - An optional list of strings containing domains that this spider is - allowed to crawl. Requests for URLs not belonging to the domain names - specified in this list (or their subdomains) won't be followed if - :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is - enabled. + .. autoattribute:: start_urls - Let's say your target url is ``https://www.example.com/1.html``, - then add ``'example.com'`` to the list. + .. autoattribute:: custom_settings - .. autoattribute:: start_urls + .. autoattribute:: crawler - .. attribute:: custom_settings + .. autoattribute:: settings - A dictionary of settings that will be overridden from the project wide - configuration when running this spider. It must be defined as a class - attribute since the settings are updated before instantiation. + .. autoattribute:: logger - For a list of available built-in settings see: - :ref:`topics-settings-ref`. + .. attribute:: state + :type: dict[str, Any] - .. attribute:: crawler + Spider state to persist between batches. + See :ref:`topics-keeping-persistent-state-between-batches` for details. - This attribute is set by the :meth:`from_crawler` class method after - initializing the class, and links to the - :class:`~scrapy.crawler.Crawler` object to which this spider instance is - bound. + .. automethod:: update_settings - Crawlers encapsulate a lot of components in the project for their single - entry access (such as extensions, middlewares, signals managers, etc). - See :ref:`topics-api-crawler` to know more about them. + .. automethod:: from_crawler - .. attribute:: settings + .. automethod:: start - Configuration for running this spider. This is a - :class:`~scrapy.settings.Settings` instance, see the - :ref:`topics-settings` topic for a detailed introduction on this subject. + .. automethod:: parse - .. attribute:: logger + .. method:: closed(reason) - Python logger created with the Spider's :attr:`name`. You can use it to - send log messages through it as described on - :ref:`topics-logging-from-spiders`. - - .. attribute:: state - - A dict you can use to persist some spider state between batches. - See :ref:`topics-keeping-persistent-state-between-batches` to know more about it. - - .. method:: from_crawler(crawler, *args, **kwargs) - - This is the class method used by Scrapy to create your spiders. - - You probably won't need to override this directly because the default - implementation acts as a proxy to the :meth:`__init__` method, calling - it with the given arguments ``args`` and named arguments ``kwargs``. - - Nonetheless, this method sets the :attr:`crawler` and :attr:`settings` - attributes in the new instance so they can be accessed later inside the - spider's code. - - .. versionchanged:: 2.11 - - The settings in ``crawler.settings`` can now be modified in this - method, which is handy if you want to modify them based on - arguments. As a consequence, these settings aren't the final values - as they can be modified later by e.g. :ref:`add-ons - `. For the same reason, most of the - :class:`~scrapy.crawler.Crawler` attributes aren't initialized at - this point. - - The final settings and the initialized - :class:`~scrapy.crawler.Crawler` attributes are available in the - :meth:`start` method, handlers of the - :signal:`engine_started` signal and later. - - :param crawler: crawler to which the spider will be bound - :type crawler: :class:`~scrapy.crawler.Crawler` instance - - :param args: arguments passed to the :meth:`__init__` method - :type args: list - - :param kwargs: keyword arguments passed to the :meth:`__init__` method - :type kwargs: dict - - .. classmethod:: update_settings(settings) - - The ``update_settings()`` method is used to modify the spider's settings - and is called during initialization of a spider instance. - - It takes a :class:`~scrapy.settings.Settings` object as a parameter and - can add or update the spider's configuration values. This method is a - class method, meaning that it is called on the :class:`~scrapy.Spider` - class and allows all instances of the spider to share the same - configuration. - - While per-spider settings can be set in - :attr:`~scrapy.Spider.custom_settings`, using ``update_settings()`` - allows you to dynamically add, remove or change settings based on other - settings, spider attributes or other factors and use setting priorities - other than ``'spider'``. Also, it's easy to extend ``update_settings()`` - in a subclass by overriding it, while doing the same with - :attr:`~scrapy.Spider.custom_settings` can be hard. - - For example, suppose a spider needs to modify :setting:`FEEDS`: - - .. code-block:: python - - import scrapy - - - class MySpider(scrapy.Spider): - name = "myspider" - custom_feed = { - "/home/user/documents/items.json": { - "format": "json", - "indent": 4, - } - } - - @classmethod - def update_settings(cls, settings): - super().update_settings(settings) - settings.setdefault("FEEDS", {}).update(cls.custom_feed) - - .. automethod:: start - - .. method:: parse(response) - - This is the default callback used by Scrapy to process downloaded - responses, when their requests don't specify a callback. - - The ``parse`` method is in charge of processing the response and returning - scraped data and/or more URLs to follow. Other Requests callbacks have - the same requirements as the :class:`~scrapy.Spider` class. - - This method, as well as any other Request callback, must return a - :class:`~scrapy.Request` object, an :ref:`item object `, an - iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects - `, or ``None``. - - :param response: the response to parse - :type response: :class:`~scrapy.http.Response` - - .. method:: closed(reason) - - Called when the spider closes. This method provides a shortcut to - signals.connect() for the :signal:`spider_closed` signal. + Called when the spider closes. This method provides a shortcut to + signals.connect() for the :signal:`spider_closed` signal. Let's see an example: @@ -461,84 +341,17 @@ with a ``TestItem`` declared in a ``myproject.items`` module: CrawlSpider ----------- -.. class:: CrawlSpider +.. autoclass:: CrawlSpider - This is the most commonly used spider for crawling regular websites, as it - provides a convenient mechanism for following links by defining a set of rules. - It may not be the best suited for your particular web sites or project, but - it's generic enough for several cases, so you can start from it and override it - as needed for more custom functionality, or just implement your own spider. + .. autoattribute:: rules - Apart from the attributes inherited from Spider (that you must - specify), this class supports a new attribute: - - .. attribute:: rules - - Which is a list of one (or more) :class:`Rule` objects. Each :class:`Rule` - defines a certain behaviour for crawling the site. Rules objects are - described below. If multiple rules match the same link, the first one - will be used, according to the order they're defined in this attribute. - - This spider also exposes an overridable method: - - .. method:: parse_start_url(response, **kwargs) - - This method is called for each response produced for the URLs in - the spider's ``start_urls`` attribute. It allows to parse - the initial responses and must return either an - :ref:`item object `, a :class:`~scrapy.Request` - object, or an iterable containing any of them. + .. automethod:: parse_start_url Crawling rules ~~~~~~~~~~~~~~ .. autoclass:: Rule - ``link_extractor`` is a :ref:`Link Extractor ` object which - defines how links will be extracted from each crawled page. Each produced link will - be used to generate a :class:`~scrapy.Request` object, which will contain the - link's text in its ``meta`` dictionary (under the ``link_text`` key). - If omitted, a default link extractor created with no arguments will be used, - resulting in all links being extracted. - - ``callback`` is a callable or a string (in which case a method from the spider - object with that name will be used) to be called for each link extracted with - the specified link extractor. This callback receives a :class:`~scrapy.http.Response` - as its first argument and must return either a single instance or an iterable of - :ref:`item objects ` and/or :class:`~scrapy.Request` objects - (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response` - object will contain the text of the link that produced the :class:`~scrapy.Request` - in its ``meta`` dictionary (under the ``link_text`` key) - - ``cb_kwargs`` is a dict containing the keyword arguments to be passed to the - callback function. - - ``follow`` is a boolean which specifies if links should be followed from each - response extracted with this rule. If ``callback`` is None ``follow`` defaults - to ``True``, otherwise it defaults to ``False``. - - ``process_links`` is a callable, or a string (in which case a method from the - spider object with that name will be used) which will be called for each list - of links extracted from each response using the specified ``link_extractor``. - This is mainly used for filtering purposes. - - ``process_request`` is a callable (or a string, in which case a method from - the spider object with that name will be used) which will be called for every - :class:`~scrapy.Request` extracted by this rule. This callable should - take said request as first argument and the :class:`~scrapy.http.Response` - from which the request originated as second argument. It must return a - ``Request`` object or ``None`` (to filter out the request). - - ``errback`` is a callable or a string (in which case a method from the spider - object with that name will be used) to be called if any exception is - raised while processing a request generated by the rule. - It receives a :class:`Twisted Failure ` - instance as first parameter. - - .. warning:: Because of its internal implementation, you must explicitly set - callbacks for new requests when writing :class:`CrawlSpider`-based spiders; - unexpected behaviour can occur otherwise. - CrawlSpider example ~~~~~~~~~~~~~~~~~~~ @@ -592,97 +405,19 @@ a dictionary will be filled with it. XMLFeedSpider ------------- -.. class:: XMLFeedSpider +.. autoclass:: XMLFeedSpider - XMLFeedSpider is designed for parsing XML feeds by iterating through them by a - certain node name. The iterator can be chosen from: ``iternodes``, ``xml``, - and ``html``. It's recommended to use the ``iternodes`` iterator for - performance reasons, since the ``xml`` and ``html`` iterators generate the - whole DOM at once in order to parse it. However, using ``html`` as the - iterator may be useful when parsing XML with bad markup. + .. autoattribute:: iterator - To set the iterator and the tag name, you must define the following class - attributes: + .. autoattribute:: itertag - .. attribute:: iterator + .. autoattribute:: namespaces - A string which defines the iterator to use. It can be either: + .. automethod:: adapt_response - - ``'iternodes'`` - a fast iterator based on ``lxml`` + .. automethod:: parse_node - - ``'html'`` - an iterator which uses :class:`~scrapy.Selector`. - Keep in mind this uses DOM parsing and must load all DOM in memory - which could be a problem for big feeds - - - ``'xml'`` - an iterator which uses :class:`~scrapy.Selector`. - Keep in mind this uses DOM parsing and must load all DOM in memory - which could be a problem for big feeds - - It defaults to: ``'iternodes'``. - - .. attribute:: itertag - - A string with the name of the node (or element) to iterate in. Example: - - .. code-block:: python - - itertag = "product" - - .. attribute:: namespaces - - A list of ``(prefix, uri)`` tuples which define the namespaces - available in that document that will be processed with this spider. The - ``prefix`` and ``uri`` will be used to automatically register - namespaces using the - :meth:`~scrapy.Selector.register_namespace` method. - - You can then specify nodes with namespaces in the :attr:`itertag` - attribute. - - Example: - - .. code-block:: python - - from scrapy.spiders import XMLFeedSpider - - - class YourSpider(XMLFeedSpider): - - namespaces = [("n", "http://www.sitemaps.org/schemas/sitemap/0.9")] - itertag = "n:url" - # ... - - Apart from these new attributes, this spider has the following overridable - methods too: - - .. method:: adapt_response(response) - - A method that receives the response as soon as it arrives from the spider - middleware, before the spider starts parsing it. It can be used to modify - the response body before parsing it. This method receives a response and - also returns a response (it could be the same or another one). - - .. method:: parse_node(response, selector) - - This method is called for the nodes matching the provided tag name - (``itertag``). Receives the response and an - :class:`~scrapy.Selector` for each node. Overriding this - method is mandatory. Otherwise, your spider won't work. This method - must return an :ref:`item object `, a - :class:`~scrapy.Request` object, or an iterable containing any of - them. - - .. method:: process_results(response, results) - - This method is called for each result (item or request) returned by the - spider, and it's intended to perform any last time processing required - before returning the results to the framework core, for example setting the - item IDs. It receives a list of results and the response which originated - those results. It must return a list of results (items or requests). - - .. warning:: Because of its internal implementation, you must explicitly set - callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders; - unexpected behaviour can occur otherwise. + .. automethod:: process_results XMLFeedSpider example @@ -722,32 +457,19 @@ prints them out, and stores some random data in an :class:`~scrapy.Item`. CSVFeedSpider ------------- -.. class:: CSVFeedSpider +.. autoclass:: CSVFeedSpider - This spider is very similar to the XMLFeedSpider, except that it iterates - over rows, instead of nodes. The method that gets called in each iteration - is :meth:`parse_row`. + .. autoattribute:: delimiter - .. attribute:: delimiter + .. autoattribute:: quotechar - A string with the separator character for each field in the CSV file - Defaults to ``','`` (comma). + .. autoattribute:: headers - .. attribute:: quotechar + .. automethod:: adapt_response - A string with the enclosure character for each field in the CSV file - Defaults to ``'"'`` (quotation mark). + .. automethod:: parse_row - .. attribute:: headers - - A list of the column names in the CSV file. - - .. method:: parse_row(response, row) - - Receives a response and a dict (representing each row) with a key for each - provided (or detected) header of the CSV file. This spider also gives the - opportunity to override ``adapt_response`` and ``process_results`` methods - for pre- and post-processing purposes. + .. automethod:: process_results CSVFeedSpider example ~~~~~~~~~~~~~~~~~~~~~ @@ -783,121 +505,17 @@ Let's see an example similar to the previous one, but using a SitemapSpider ------------- -.. class:: SitemapSpider +.. autoclass:: SitemapSpider - SitemapSpider allows you to crawl a site by discovering the URLs using - `Sitemaps`_. + .. autoattribute:: sitemap_urls - It supports nested sitemaps and discovering sitemap urls from - `robots.txt`_. + .. autoattribute:: sitemap_rules - .. attribute:: sitemap_urls + .. autoattribute:: sitemap_follow - A list of urls pointing to the sitemaps whose urls you want to crawl. + .. autoattribute:: sitemap_alternate_links - You can also point to a `robots.txt`_ and it will be parsed to extract - sitemap urls from it. - - .. attribute:: sitemap_rules - - A list of tuples ``(regex, callback)`` where: - - * ``regex`` is a regular expression to match urls extracted from sitemaps. - ``regex`` can be either a str or a compiled regex object. - - * callback is the callback to use for processing the urls that match - the regular expression. ``callback`` can be a string (indicating the - name of a spider method) or a callable. - - For example: - - .. code-block:: python - - sitemap_rules = [("/product/", "parse_product")] - - Rules are applied in order, and only the first one that matches will be - used. - - If you omit this attribute, all urls found in sitemaps will be - processed with the ``parse`` callback. - - .. attribute:: sitemap_follow - - A list of regexes of sitemap that should be followed. This is only - for sites that use `Sitemap index files`_ that point to other sitemap - files. - - By default, all sitemaps are followed. - - .. attribute:: sitemap_alternate_links - - Specifies if alternate links for one ``url`` should be followed. These - are links for the same website in another language passed within - the same ``url`` block. - - For example: - - .. code-block:: xml - - - http://example.com/ - - - - With ``sitemap_alternate_links`` set, this would retrieve both URLs. With - ``sitemap_alternate_links`` disabled, only ``http://example.com/`` would be - retrieved. - - Default is ``sitemap_alternate_links`` disabled. - - .. method:: sitemap_filter(entries) - - This is a filter function that could be overridden to select sitemap entries - based on their attributes. - - For example: - - .. code-block:: xml - - - http://example.com/ - 2005-01-01 - - - We can define a ``sitemap_filter`` function to filter ``entries`` by date: - - .. code-block:: python - - from datetime import datetime - from scrapy.spiders import SitemapSpider - - - class FilteredSitemapSpider(SitemapSpider): - name = "filtered_sitemap_spider" - allowed_domains = ["example.com"] - sitemap_urls = ["http://example.com/sitemap.xml"] - - def sitemap_filter(self, entries): - for entry in entries: - date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") - if date_time.year >= 2005: - yield entry - - This would retrieve only ``entries`` modified on 2005 and the following - years. - - Entries are dict objects extracted from the sitemap document. - Usually, the key is the tag name and the value is the text inside it. - - It's important to notice that: - - - as the loc attribute is required, entries without this tag are discarded - - alternate links are stored in a list with the key ``alternate`` - (see ``sitemap_alternate_links``) - - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname`` - - If you omit this method, all entries found in sitemaps will be - processed, observing other attributes and their settings. + .. automethod:: sitemap_filter SitemapSpider examples @@ -986,7 +604,5 @@ Combine SitemapSpider with other sources of urls: .. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/params.html .. _Sitemaps: https://www.sitemaps.org/index.html -.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index .. _robots.txt: https://www.robotstxt.org/ -.. _TLD: https://en.wikipedia.org/wiki/Top-level_domain .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/ diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 02dfa2ac6..21e3fbfea 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -36,14 +36,70 @@ class Spider(object_ref): It provides a default :meth:`start` implementation that sends requests based on the :attr:`start_urls` class attribute and calls the :meth:`parse` method for each response. + + Like :ref:`Scrapy components `, spiders are + :ref:`initialized from the crawler `, through + :meth:`~scrapy.Spider.from_crawler`, and can be :ref:`configured through + settings `, which they may also override through + :attr:`custom_settings` or :meth:`~scrapy.Spider.update_settings`. """ + #: The name of this spider. + #: + #: Every spider needs one: :class:`Spider` raises :exc:`ValueError` on + #: initialization if it has no name. You usually define it as a class + #: attribute, but you can also pass it at initialization time instead, e.g. + #: ``CrawlerProcess.crawl(MySpider, name="myspider")`` when :ref:`running + #: Scrapy from a script `. + #: + #: The name is also how Scrapy locates a spider: the default :ref:`spider + #: loader ` indexes the spiders of your project by + #: name, which is what allows the :command:`crawl` command to find them, + #: and the :command:`runspider` command ignores spider classes that have no + #: name. Names should hence be unique within a project; the default spider + #: loader warns about duplicates and keeps only one of the matching spider + #: classes. Nothing prevents you from running more than one instance of the + #: same spider, though, and a custom spider loader (see + #: :setting:`SPIDER_LOADER_CLASS`) may map names to spider classes in a + #: completely different way. + #: + #: If the spider scrapes a single domain, a common practice is to name the + #: spider after that domain, replacing dots with underscores. For example, a + #: spider that crawls ``books.toscrape.com`` would often be called + #: ``books_toscrape_com``. name: str + + #: Settings that override the project-wide configuration when running this + #: spider. It must be defined as a class attribute, since the settings are + #: updated before instantiation. + #: + #: See :ref:`topics-settings-ref` for a list of built-in settings. + #: + #: .. seealso:: :meth:`~scrapy.Spider.update_settings`, a more verbose but + #: more flexible alternative, which allows setting values based on other + #: settings or on spider attributes, using priorities other than + #: ``'spider'``, and extending the settings of a base spider class. + #: + #: :ref:`spider-settings` custom_settings: dict[str, Any] | None = None #: Start URLs. See :meth:`start`. start_urls: list[str] + #: This attribute is set by the :meth:`~scrapy.Spider.from_crawler` class + #: method after initializing the class, and links to the + #: :class:`~scrapy.crawler.Crawler` object to which this spider instance is + #: bound. + #: + #: Crawlers encapsulate a lot of components in the project for their single + #: entry access (such as extensions, middlewares, signals managers, etc). + #: See :ref:`topics-api-crawler` for details. + crawler: Crawler + + #: Configuration for running this spider. + #: See :ref:`topics-settings` for details. + settings: BaseSettings + def __init__(self, name: str | None = None, **kwargs: Any): if name is not None: self.name: str = name @@ -55,6 +111,11 @@ class Spider(object_ref): @property def logger(self) -> SpiderLoggerAdapter: + """Python logger created with the spider's :attr:`name`. + + Use it to send log messages. See :ref:`topics-logging-from-spiders` for + details. + """ # circular import from scrapy.utils.log import SpiderLoggerAdapter # noqa: PLC0415 @@ -77,13 +138,42 @@ class Spider(object_ref): @classmethod def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: + """Return a new spider instance bound to *crawler*. + + You probably won't need to override this directly because the default + implementation acts as a proxy to the ``__init__()`` method, calling + it with the given arguments *args* and named arguments *kwargs*, which + is how :ref:`spider arguments ` reach a spider. + + Nonetheless, this method sets the :attr:`crawler` and :attr:`settings` + attributes in the new instance so they can be accessed later inside the + spider's code. + + .. seealso:: :ref:`from-crawler` + + .. versionchanged:: 2.11 + + The settings in ``crawler.settings`` can now be modified in this + method, which is handy if you want to modify them based on + arguments. As a consequence, these settings aren't the final values + as they can be modified later by e.g. :ref:`add-ons + `. For the same reason, most of the + :class:`~scrapy.crawler.Crawler` attributes aren't initialized at + this point. + + The settings are final and those + :class:`~scrapy.crawler.Crawler` attributes are initialized by the + time the :meth:`start` method runs and the :signal:`engine_started` + signal is sent, which is the earliest point where your spider code + can rely on them. + """ spider = cls(*args, **kwargs) spider._set_crawler(crawler) return spider def _set_crawler(self, crawler: Crawler) -> None: - self.crawler: Crawler = crawler - self.settings: BaseSettings = crawler.settings + self.crawler = crawler + self.settings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) async def start(self) -> AsyncIterator[Any]: @@ -143,12 +233,58 @@ class Spider(object_ref): else: def parse(self, response: Response, **kwargs: Any) -> Any: + """Handle *response*, returning scraped data and/or more URLs to + follow. + + Scrapy calls this method for the responses of requests that do not + define a callback. Other request callbacks have the same + requirements as this method. + + It must return a :class:`~scrapy.Request` object, an :ref:`item + object `, an iterable of :class:`~scrapy.Request` + objects and/or :ref:`item objects `, or ``None``. + """ raise NotImplementedError( f"{self.__class__.__name__}.parse callback is not defined" ) @classmethod def update_settings(cls, settings: BaseSettings) -> None: + """Modify *settings*, the settings of the spider. + + This method is called during the initialization of a spider instance. + It can add or update the spider's configuration values. It is a class + method, meaning that it is called on the :class:`~scrapy.Spider` class + and allows all instances of the spider to share the same configuration. + + While per-spider settings can be set in :attr:`custom_settings`, using + this method allows you to dynamically add, remove or change settings + based on other settings, spider attributes or other factors, and to use + setting priorities other than ``'spider'``. Also, it's easy to extend + this method in a subclass by overriding it, while doing the same with + :attr:`custom_settings` can be hard. + + For example, suppose a spider needs to modify :setting:`FEEDS`: + + .. code-block:: python + + import scrapy + + + class MySpider(scrapy.Spider): + name = "myspider" + custom_feed = { + "/home/user/documents/items.json": { + "format": "json", + "indent": 4, + } + } + + @classmethod + def update_settings(cls, settings): + super().update_settings(settings) + settings.setdefault("FEEDS", {}).update(cls.custom_feed) + """ settings.setdict(cls.custom_settings or {}, priority="spider") @classmethod diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 373c0b8b5..3cc8d20ad 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -61,6 +61,51 @@ _default_link_extractor = LinkExtractor() class Rule: + """A link-following rule for :class:`CrawlSpider`, to be used in its + :attr:`~scrapy.spiders.CrawlSpider.rules` attribute. + + It defines which links to extract from a response, what to do with the + responses those links produce, and whether to keep extracting links from + them. + + *callback*, *process_links*, *process_request* and *errback* may be + specified as a string instead of a callable, in which case the spider method + with that name is used. + + *link_extractor* is a :class:`LinkExtractor + ` object which defines + how links will be extracted from each crawled page. Each produced link will + be used to generate a :class:`~scrapy.Request` object, which will contain + the link's text in its ``meta`` dictionary (under the ``link_text`` key). If + omitted, a default link extractor created with no arguments will be used, + resulting in all links being extracted. + + *callback* is called for each link extracted with *link_extractor*. It is a + :meth:`spider callback `. If omitted, no callback runs + for the matching responses, which are then only used to extract further + links, as per *follow*. + + *cb_kwargs* are the keyword arguments to pass to the callback. + + *follow* determines whether links should be followed from each response + extracted with this rule. It defaults to ``True`` if *callback* is ``None``, + and to ``False`` otherwise. + + *process_links* is called for each list of links extracted from each + response using *link_extractor*. This is mainly used for filtering purposes. + + *process_request* is called for every :class:`~scrapy.Request` extracted by + this rule. It should take said request as first argument and the + :class:`~scrapy.http.Response` from which the request originated as second + argument. It must return a :class:`~scrapy.Request` object or ``None`` (to + filter out the request). + + *errback* is called if any exception is raised while processing a request + generated by the rule. It receives a + :class:`Twisted Failure ` instance as first + parameter. + """ + def __init__( self, link_extractor: LinkExtractor | None = None, @@ -96,6 +141,34 @@ class Rule: class CrawlSpider(Spider): + """Spider that follows links based on a set of rules. + + You declare a set of :class:`Rule` objects in :attr:`rules`, and this spider + extracts links from every response it gets and follows them accordingly. + + Use it only for crawls that fit that declarative model. As soon as you need + to control which requests are sent, in which order, or which data they carry + along, subclass :class:`~scrapy.Spider` instead and follow links from your + callbacks, e.g. with :meth:`response.follow + `. That is usually simpler than bending + :attr:`rules` to a use case they do not cover. + + .. warning:: Unlike in other spiders, a request without a callback is not + handled by :meth:`Spider.parse `. It is handled by + :meth:`parse_start_url`, and its links are followed according to + :attr:`rules`. + + That is the way to feed a response of your own to :attr:`rules`. If you + want your own parsing code to run for it instead, set its callback + explicitly; defining :meth:`~scrapy.Spider.parse` is not enough. + + Requests generated from :attr:`rules` are not affected, they always get a + callback of their own. + """ + + #: The rules that define how to crawl the site. If multiple rules match the + #: same link, the first one will be used, according to the order they're + #: defined in this attribute. rules: Sequence[Rule] = () _rules: list[Rule] _follow_links: bool @@ -123,6 +196,12 @@ class CrawlSpider(Spider): ) def parse_start_url(self, response: Response, **kwargs: Any) -> Any: + """Handle *response*, one of the responses produced for the URLs in the + :attr:`~scrapy.Spider.start_urls` attribute of the spider. + + This is a :meth:`spider callback `; override it to + parse the initial responses. + """ return () def process_results( diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 1e7ac9c34..6377ce36a 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -21,46 +21,97 @@ if TYPE_CHECKING: class XMLFeedSpider(Spider): - """ - This class intends to be the base class for spiders that scrape - from XML feeds. + """Spider for parsing XML feeds by iterating through them by a certain node + name. - You can choose whether to parse the file using the 'iternodes' iterator, an - 'xml' selector, or an 'html' selector. In most cases, it's convenient to - use iternodes, since it's a faster and cleaner. + The iterator can be chosen from: ``iternodes``, ``xml``, and ``html``. It's + recommended to use the ``iternodes`` iterator for performance reasons, since + the ``xml`` and ``html`` iterators generate the whole DOM at once in order to + parse it. However, using ``html`` as the iterator may be useful when parsing + XML with bad markup. + + To set the iterator and the tag name, you must define the :attr:`iterator` + and :attr:`itertag` class attributes. + + .. warning:: Unlike in other spiders, a request without a callback is not + handled by :meth:`Spider.parse `. Its response is + parsed as an XML feed, calling :meth:`parse_node` for each matching node. + + That is the way to have an additional feed parsed as such. If you want + your own parsing code to run for it instead, set its callback explicitly; + defining :meth:`~scrapy.Spider.parse` is not enough. """ + #: The iterator to use. It can be either: + #: + #: - ``'iternodes'`` - a fast iterator based on ``lxml`` + #: + #: - ``'html'`` - an iterator which uses :class:`~scrapy.Selector`. + #: Keep in mind this uses DOM parsing and must load all DOM in memory + #: which could be a problem for big feeds + #: + #: - ``'xml'`` - an iterator which uses :class:`~scrapy.Selector`. + #: Keep in mind this uses DOM parsing and must load all DOM in memory + #: which could be a problem for big feeds iterator: str = "iternodes" + + #: Name of the node (or element) to iterate in. Example: + #: + #: .. code-block:: python + #: + #: itertag = "product" itertag: str = "item" + + #: ``(prefix, uri)`` tuples defining the namespaces available in that + #: document that will be processed with this spider. The ``prefix`` and + #: ``uri`` will be used to automatically register namespaces using the + #: :meth:`~scrapy.Selector.register_namespace` method. + #: + #: You can then specify nodes with namespaces in the :attr:`itertag` + #: attribute. + #: + #: Example: + #: + #: .. code-block:: python + #: + #: from scrapy.spiders import XMLFeedSpider + #: + #: + #: class YourSpider(XMLFeedSpider): + #: + #: namespaces = [("n", "http://www.sitemaps.org/schemas/sitemap/0.9")] + #: itertag = "n:url" + #: # ... namespaces: Sequence[tuple[str, str]] = () def process_results( self, response: Response, results: Iterable[Any] ) -> Iterable[Any]: - """This overridable method is called for each result (item or request) - returned by the spider, and it's intended to perform any last time - processing required before returning the results to the framework core, - for example setting the item GUIDs. It receives a list of results and - the response which originated that results. It must return a list of - results (items or requests). + """Handle *results*, the items and requests returned by the spider for + *response*, and return them, either unmodified or with changes. + + Override it to perform any last-time processing required before + returning the results to the framework core, for example setting the + item IDs. """ return results def adapt_response(self, response: Response) -> Response: - """You can override this function in order to make any changes you want - to into the feed before parsing it. This function must return a - response. + """Handle *response* as soon as it arrives from the spider middleware, + before the spider starts parsing it, and return a response, which can be + the same one or a different one. + + Override it to make any changes you want to the feed before parsing it, + e.g. to its body. """ return response def parse_node(self, response: Response, selector: Selector) -> Any: - """This method is called for the nodes matching the provided tag name - (itertag). Receives the response and an Selector for each node. + """Handle a node of *response* matching :attr:`itertag`, for which + *selector* is a :class:`~scrapy.Selector`. - This method must return either an item, a request, or a list - containing any of them. - - This method must be overridden with your custom spider functionality. + This is a :meth:`spider callback `. Overriding it + is mandatory. Otherwise, your spider won't work. """ if hasattr(self, "parse_item"): # backward compatibility return self.parse_item(response, selector) @@ -109,39 +160,50 @@ class XMLFeedSpider(Spider): class CSVFeedSpider(Spider): """Spider for parsing CSV feeds. - It receives a CSV file in a response; iterates through each of its rows, - and calls parse_row with a dict containing each field's data. - This spider also gives the opportunity to override adapt_response and - process_results methods for pre and post-processing purposes. + This spider is very similar to :class:`XMLFeedSpider`, except that it + iterates over rows, instead of nodes. The method that gets called in each + iteration is :meth:`parse_row`. - You can set some options regarding the CSV file, such as the delimiter, quotechar - and the file's headers. + .. warning:: Unlike in other spiders, a request without a callback is not + handled by :meth:`Spider.parse `. Its response is + parsed as a CSV feed, calling :meth:`parse_row` for each row. + + That is the way to have an additional feed parsed as such. If you want + your own parsing code to run for it instead, set its callback explicitly; + defining :meth:`~scrapy.Spider.parse` is not enough. """ - delimiter: str | None = ( - None # When this is None, python's csv module's default delimiter is used - ) - quotechar: str | None = ( - None # When this is None, python's csv module's default quotechar is used - ) + #: Separator character for each field in the CSV file. + #: + #: ``None`` means using the default delimiter of the :mod:`csv` module, + #: ``','`` (comma). + delimiter: str | None = None + + #: Enclosure character for each field in the CSV file. + #: + #: ``None`` means using the default quote character of the :mod:`csv` + #: module, ``'"'`` (quotation mark). + quotechar: str | None = None + + #: Column names in the CSV file. headers: list[str] | None = None def process_results( self, response: Response, results: Iterable[Any] ) -> Iterable[Any]: - """This method has the same purpose as the one in XMLFeedSpider""" + """Same as :meth:`XMLFeedSpider.process_results`.""" return results def adapt_response(self, response: Response) -> Response: - """This method has the same purpose as the one in XMLFeedSpider""" + """Same as :meth:`XMLFeedSpider.adapt_response`.""" return response def parse_row(self, response: Response, row: dict[str, str]) -> Any: - """Receives a response and a dict (representing each row) with a key for - each provided (or detected) header of the CSV file. + """Handle *row*, a row of *response* as a dict with a key for each + provided (or detected) header of the CSV file. - This method must be overridden with your custom spider functionality. + Overriding this method is mandatory. Otherwise, your spider won't work. """ raise NotImplementedError diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 2a80b8d24..a17929edc 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -24,11 +24,66 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): + """Spider that crawls a site by discovering its URLs using `sitemaps + `_. + + It supports nested sitemaps and discovering sitemap URLs from `robots.txt + `_. + """ + + #: URLs pointing to the sitemaps whose URLs you want to crawl. + #: + #: You can also point to a `robots.txt `_ and it + #: will be parsed to extract sitemap URLs from it. sitemap_urls: Sequence[str] = () + + #: ``(regex, callback)`` tuples where: + #: + #: - ``regex`` is a regular expression to match URLs extracted from + #: sitemaps. ``regex`` can be either a str or a compiled regex object. + #: + #: - ``callback`` is the callback to use for processing the URLs that match + #: the regular expression. ``callback`` can be a string (indicating the + #: name of a spider method) or a callable. + #: + #: For example: + #: + #: .. code-block:: python + #: + #: sitemap_rules = [("/product/", "parse_product")] + #: + #: Rules are applied in order, and only the first one that matches will be + #: used. + #: + #: The default value makes all URLs found in sitemaps be processed with the + #: :meth:`~scrapy.Spider.parse` callback. sitemap_rules: Sequence[tuple[re.Pattern[str] | str, str | CallbackT]] = [ ("", "parse") ] + + #: Regexes of sitemaps that should be followed. This is only for sites that + #: use `sitemap index files + #: `_ that point to other + #: sitemap files. + #: + #: By default, all sitemaps are followed. sitemap_follow: Sequence[re.Pattern[str] | str] = [""] + + #: Specifies if alternate links for one ``url`` should be followed. These are + #: links for the same website in another language passed within the same + #: ``url`` block. + #: + #: For example: + #: + #: .. code-block:: xml + #: + #: + #: http://example.com/ + #: + #: + #: + #: When enabled, this would retrieve both URLs. When disabled, only + #: ``http://example.com/`` would be retrieved. sitemap_alternate_links: bool = False _max_size: int _warn_size: int @@ -60,9 +115,54 @@ class SitemapSpider(Spider): def sitemap_filter( self, entries: Iterable[dict[str, Any]] ) -> Iterable[dict[str, Any]]: - """This method can be used to filter sitemap entries by their - attributes, for example, you can filter locs with lastmod greater - than a given date (see docs). + """Yield the sitemap entries from *entries* that should be processed. + + Override it to select sitemap entries based on their attributes. For + example, given the following sitemap entry: + + .. code-block:: xml + + + http://example.com/ + 2005-01-01 + + + You can filter entries by date as follows: + + .. code-block:: python + + from datetime import datetime + from scrapy.spiders import SitemapSpider + + + class FilteredSitemapSpider(SitemapSpider): + name = "filtered_sitemap_spider" + allowed_domains = ["example.com"] + sitemap_urls = ["http://example.com/sitemap.xml"] + + def sitemap_filter(self, entries): + for entry in entries: + date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") + if date_time.year >= 2005: + yield entry + + This would retrieve only entries modified on 2005 and the following + years. + + Entries are dict objects extracted from the sitemap document. Usually, + the key is the tag name and the value is the text inside it. + + It's important to notice that: + + - as the ``loc`` attribute is required, entries without this tag are + discarded + - alternate links are stored in a list with the key ``alternate`` + (see :attr:`sitemap_alternate_links`) + - namespaces are removed, so lxml tags named as ``{namespace}tagname`` + become only ``tagname`` + + The default implementation yields all entries, observing other + attributes and their settings. """ yield from entries