Merge aa57a07a82 into ad43bf0c56

docs/topics/spiders.rst

@@ -862,6 +862,30 @@ SitemapSpider
 
         Default is ``sitemap_alternate_links`` disabled.
 
+    .. attribute:: sitemap_meta
+
+        Sitemap entry keys to copy into the :attr:`~scrapy.Request.meta` of the
+        requests generated from those entries, so that callbacks can read
+        sitemap data such as ``lastmod``:
+
+        .. versionadded:: VERSION
+
+        .. code-block:: python
+
+            sitemap_meta = {"lastmod"}
+
+
+            def parse(self, response):
+                yield {"lastmod": response.meta.get("lastmod")}
+
+        Use a dict to store an entry key under a different meta key, e.g.
+        ``sitemap_meta = {"lastmod": "sitemap_lastmod"}``.
+
+        See :meth:`sitemap_filter` for the available entry keys. Keys missing
+        from an entry are skipped.
+
+        By default no sitemap data is copied.
+
     .. method:: sitemap_filter(entries)
 
         This is a filter function that could be overridden to select sitemap entries
@@ -901,12 +925,20 @@ SitemapSpider
         Entries are dict objects extracted from the sitemap document.
         Usually, the key is the tag name and the value is the text inside it.
 
+        The `sitemaps protocol`_ defines ``loc``, ``lastmod``, ``changefreq``
+        and ``priority`` for ``urlset`` entries, and ``loc`` and ``lastmod``
+        for ``sitemapindex`` entries. Sites may use additional tags, such as
+        those of the image, video and news sitemap extensions, which become
+        keys as well.
+
         It's important to notice that:
 
         - as the loc attribute is required, entries without this tag are discarded
         - alternate links are stored in a list with the key ``alternate``
           (see ``sitemap_alternate_links``)
         - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname``
+        - the value is the text of the tag, so a tag with nested tags, such as
+          ``<image:image>``, gets an empty string
 
         If you omit this method, all entries found in sitemaps will be
         processed, observing other attributes and their settings.
@@ -999,6 +1031,7 @@ Combine SitemapSpider with other sources of urls:
 .. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/params.html
 .. _Sitemaps: https://www.sitemaps.org/index.html
 .. _Sitemap index files: https://www.sitemaps.org/protocol.html#index
+.. _sitemaps protocol: https://www.sitemaps.org/protocol.html
 .. _robots.txt: https://www.robotstxt.org/
 .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain
 .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/

scrapy/spiders/sitemap.py

@@ -4,7 +4,7 @@ import logging
 import re
 
 # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation
-from collections.abc import AsyncIterator, Iterable, Sequence  # noqa: TC003
+from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, cast
 
 from scrapy.http import Request, Response, XmlResponse
@@ -30,6 +30,7 @@ class SitemapSpider(Spider):
     ]
     sitemap_follow: Sequence[re.Pattern[str] | str] = [""]
     sitemap_alternate_links: bool = False
+    sitemap_meta: Mapping[str, str] | Iterable[str] = ()
     _max_size: int
     _warn_size: int
 
@@ -52,6 +53,11 @@ class SitemapSpider(Spider):
                 c = cast("CallbackT", getattr(self, c))  # noqa: PLW2901
             self._cbs.append((regex(r), c))
         self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]
+        self._meta_keys: list[tuple[str, str]] = list(
+            self.sitemap_meta.items()
+            if isinstance(self.sitemap_meta, Mapping)
+            else ((key, key) for key in self.sitemap_meta)
+        )
 
     async def start(self) -> AsyncIterator[Any]:
         for url in self.sitemap_urls:
@@ -87,10 +93,13 @@ class SitemapSpider(Spider):
             return (Request(loc, callback=self._parse_sitemap) for loc in urls)
 
         if s.type == "urlset":
-            url_callback_pairs = list(
-                self._get_urls_and_callbacks_from_urlset(self.sitemap_filter(s))
+            url_callback_meta_triples = list(
+                self._get_urls_callbacks_and_meta_from_urlset(self.sitemap_filter(s))
+            )
+            return (
+                Request(loc, callback=c, meta=meta)
+                for loc, c, meta in url_callback_meta_triples
             )
-            return (Request(loc, callback=c) for loc, c in url_callback_pairs)
 
         logger.warning(
             "Ignoring invalid sitemap: %(response)s",
@@ -107,14 +116,20 @@ class SitemapSpider(Spider):
             if any(x.search(loc) for x in self._follow):
                 yield loc
 
-    def _get_urls_and_callbacks_from_urlset(
+    def _get_urls_callbacks_and_meta_from_urlset(
         self, it: Iterable[dict[str, Any]]
-    ) -> Iterable[tuple[str, CallbackT]]:
-        for loc in iterloc(it, self.sitemap_alternate_links):
-            for r, c in self._cbs:
-                if r.search(loc):
-                    yield loc, c
-                    break
+    ) -> Iterable[tuple[str, CallbackT, dict[str, Any] | None]]:
+        for entry in it:
+            meta = {
+                meta_key: entry[key]
+                for key, meta_key in self._meta_keys
+                if key in entry
+            } or None
+            for loc in iterloc((entry,), self.sitemap_alternate_links):
+                for r, c in self._cbs:
+                    if r.search(loc):
+                        yield loc, c, meta
+                        break
 
     def _get_sitemap_body(self, response: Response) -> bytes | None:
         """Return the sitemap body contained in the given response,

tests/test_spider_sitemap.py

@@ -20,6 +20,8 @@ from tests.utils.crawl import crawl_items
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable, Mapping
+
     from tests.mockserver.http import MockServer
 
 
@@ -315,6 +317,75 @@ Sitemap: /sitemap-relative-url.xml
         items, _ = await crawl_items(_Spider, mockserver)
         assert items == [{"url": mockserver.url("/text")}]
 
+    @pytest.mark.parametrize(
+        ("entry_keys", "expected_meta"),
+        [
+            ((), {}),
+            (
+                {"lastmod", "changefreq"},
+                {"lastmod": "2005-01-01", "changefreq": "daily"},
+            ),
+            ({"lastmod": "sitemap_lastmod"}, {"sitemap_lastmod": "2005-01-01"}),
+        ],
+    )
+    @coroutine_test
+    async def test_sitemap_meta(
+        self,
+        entry_keys: Mapping[str, str] | Iterable[str],
+        expected_meta: dict[str, str],
+        mockserver: MockServer,
+    ):
+        meta_keys = {"lastmod", "changefreq", "sitemap_lastmod"}
+
+        class _Spider(RawSitemapSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            sitemap_meta = entry_keys
+
+            def parse(self, response):
+                yield {k: v for k, v in response.meta.items() if k in meta_keys}
+
+            def raw_body(self):
+                loc = self.mockserver.url("/text")
+                return (
+                    '<?xml version="1.0" encoding="UTF-8"?>'
+                    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+                    f"<url><loc>{loc}</loc><lastmod>2005-01-01</lastmod>"
+                    "<changefreq>daily</changefreq><priority>0.8</priority></url>"
+                    "</urlset>"
+                )
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert items == [expected_meta]
+
+    @coroutine_test
+    async def test_sitemap_meta_alternate_links(self, mockserver: MockServer):
+        # Alternate links get the meta of the entry that declares them.
+        class _Spider(RawSitemapSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            sitemap_alternate_links = True
+            sitemap_meta = {"lastmod"}
+
+            def parse(self, response):
+                yield {"url": response.url, "lastmod": response.meta.get("lastmod")}
+
+            def raw_body(self):
+                loc = self.mockserver.url("/text")
+                alt = self.mockserver.url("/text?alt")
+                return (
+                    '<?xml version="1.0" encoding="UTF-8"?>'
+                    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"'
+                    ' xmlns:xhtml="http://www.w3.org/1999/xhtml">'
+                    f"<url><loc>{loc}</loc><lastmod>2005-01-01</lastmod>"
+                    f'<xhtml:link rel="alternate" hreflang="de" href="{alt}"/></url>'
+                    f"<url><loc>{self.mockserver.url('/text?no-lastmod')}</loc></url>"
+                    "</urlset>"
+                )
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert sorted(items, key=lambda item: item["url"]) == [
+            {"url": mockserver.url("/text"), "lastmod": "2005-01-01"},
+            {"url": mockserver.url("/text?alt"), "lastmod": "2005-01-01"},
+            {"url": mockserver.url("/text?no-lastmod"), "lastmod": None},
+        ]
+
     def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None:
         r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")
         spider = self.spider_class("example.com")