diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 95c80d5dc..7646392ef 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -862,6 +862,30 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. + .. attribute:: sitemap_meta + + Sitemap entry keys to copy into the :attr:`~scrapy.Request.meta` of the + requests generated from those entries, so that callbacks can read + sitemap data such as ``lastmod``: + + .. versionadded:: VERSION + + .. code-block:: python + + sitemap_meta = {"lastmod"} + + + def parse(self, response): + yield {"lastmod": response.meta.get("lastmod")} + + Use a dict to store an entry key under a different meta key, e.g. + ``sitemap_meta = {"lastmod": "sitemap_lastmod"}``. + + See :meth:`sitemap_filter` for the available entry keys. Keys missing + from an entry are skipped. + + By default no sitemap data is copied. + .. method:: sitemap_filter(entries) This is a filter function that could be overridden to select sitemap entries @@ -901,12 +925,20 @@ SitemapSpider Entries are dict objects extracted from the sitemap document. Usually, the key is the tag name and the value is the text inside it. + The `sitemaps protocol`_ defines ``loc``, ``lastmod``, ``changefreq`` + and ``priority`` for ``urlset`` entries, and ``loc`` and ``lastmod`` + for ``sitemapindex`` entries. Sites may use additional tags, such as + those of the image, video and news sitemap extensions, which become + keys as well. + It's important to notice that: - as the loc attribute is required, entries without this tag are discarded - alternate links are stored in a list with the key ``alternate`` (see ``sitemap_alternate_links``) - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname`` + - the value is the text of the tag, so a tag with nested tags, such as + ````, gets an empty string If you omit this method, all entries found in sitemaps will be processed, observing other attributes and their settings. @@ -999,6 +1031,7 @@ Combine SitemapSpider with other sources of urls: .. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/params.html .. _Sitemaps: https://www.sitemaps.org/index.html .. _Sitemap index files: https://www.sitemaps.org/protocol.html#index +.. _sitemaps protocol: https://www.sitemaps.org/protocol.html .. _robots.txt: https://www.robotstxt.org/ .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 2a80b8d24..4ddc5f21f 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -4,7 +4,7 @@ import logging import re # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation -from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 +from collections.abc import AsyncIterator, Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse @@ -30,6 +30,7 @@ class SitemapSpider(Spider): ] sitemap_follow: Sequence[re.Pattern[str] | str] = [""] sitemap_alternate_links: bool = False + sitemap_meta: Mapping[str, str] | Iterable[str] = () _max_size: int _warn_size: int @@ -52,6 +53,11 @@ class SitemapSpider(Spider): c = cast("CallbackT", getattr(self, c)) # noqa: PLW2901 self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] + self._meta_keys: list[tuple[str, str]] = list( + self.sitemap_meta.items() + if isinstance(self.sitemap_meta, Mapping) + else ((key, key) for key in self.sitemap_meta) + ) async def start(self) -> AsyncIterator[Any]: for url in self.sitemap_urls: @@ -87,10 +93,13 @@ class SitemapSpider(Spider): return (Request(loc, callback=self._parse_sitemap) for loc in urls) if s.type == "urlset": - url_callback_pairs = list( - self._get_urls_and_callbacks_from_urlset(self.sitemap_filter(s)) + url_callback_meta_triples = list( + self._get_urls_callbacks_and_meta_from_urlset(self.sitemap_filter(s)) + ) + return ( + Request(loc, callback=c, meta=meta) + for loc, c, meta in url_callback_meta_triples ) - return (Request(loc, callback=c) for loc, c in url_callback_pairs) logger.warning( "Ignoring invalid sitemap: %(response)s", @@ -107,14 +116,20 @@ class SitemapSpider(Spider): if any(x.search(loc) for x in self._follow): yield loc - def _get_urls_and_callbacks_from_urlset( + def _get_urls_callbacks_and_meta_from_urlset( self, it: Iterable[dict[str, Any]] - ) -> Iterable[tuple[str, CallbackT]]: - for loc in iterloc(it, self.sitemap_alternate_links): - for r, c in self._cbs: - if r.search(loc): - yield loc, c - break + ) -> Iterable[tuple[str, CallbackT, dict[str, Any] | None]]: + for entry in it: + meta = { + meta_key: entry[key] + for key, meta_key in self._meta_keys + if key in entry + } or None + for loc in iterloc((entry,), self.sitemap_alternate_links): + for r, c in self._cbs: + if r.search(loc): + yield loc, c, meta + break def _get_sitemap_body(self, response: Response) -> bytes | None: """Return the sitemap body contained in the given response, diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index 2f1ccab81..c4c7f432e 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -20,6 +20,8 @@ from tests.utils.crawl import crawl_items from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + from tests.mockserver.http import MockServer @@ -315,6 +317,75 @@ Sitemap: /sitemap-relative-url.xml items, _ = await crawl_items(_Spider, mockserver) assert items == [{"url": mockserver.url("/text")}] + @pytest.mark.parametrize( + ("entry_keys", "expected_meta"), + [ + ((), {}), + ( + {"lastmod", "changefreq"}, + {"lastmod": "2005-01-01", "changefreq": "daily"}, + ), + ({"lastmod": "sitemap_lastmod"}, {"sitemap_lastmod": "2005-01-01"}), + ], + ) + @coroutine_test + async def test_sitemap_meta( + self, + entry_keys: Mapping[str, str] | Iterable[str], + expected_meta: dict[str, str], + mockserver: MockServer, + ): + meta_keys = {"lastmod", "changefreq", "sitemap_lastmod"} + + class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc] + sitemap_meta = entry_keys + + def parse(self, response): + yield {k: v for k, v in response.meta.items() if k in meta_keys} + + def raw_body(self): + loc = self.mockserver.url("/text") + return ( + '' + '' + f"{loc}2005-01-01" + "daily0.8" + "" + ) + + items, _ = await crawl_items(_Spider, mockserver) + assert items == [expected_meta] + + @coroutine_test + async def test_sitemap_meta_alternate_links(self, mockserver: MockServer): + # Alternate links get the meta of the entry that declares them. + class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc] + sitemap_alternate_links = True + sitemap_meta = {"lastmod"} + + def parse(self, response): + yield {"url": response.url, "lastmod": response.meta.get("lastmod")} + + def raw_body(self): + loc = self.mockserver.url("/text") + alt = self.mockserver.url("/text?alt") + return ( + '' + '' + f"{loc}2005-01-01" + f'' + f"{self.mockserver.url('/text?no-lastmod')}" + "" + ) + + items, _ = await crawl_items(_Spider, mockserver) + assert sorted(items, key=lambda item: item["url"]) == [ + {"url": mockserver.url("/text"), "lastmod": "2005-01-01"}, + {"url": mockserver.url("/text?alt"), "lastmod": "2005-01-01"}, + {"url": mockserver.url("/text?no-lastmod"), "lastmod": None}, + ] + def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None: r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"") spider = self.spider_class("example.com")