diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a4723d01d..af9629477 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -823,6 +823,30 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. + .. attribute:: sitemap_meta + + Sitemap entry keys to copy into the :attr:`~scrapy.Request.meta` of the + requests generated from those entries, so that callbacks can read + sitemap data such as ``lastmod``: + + .. versionadded:: VERSION + + .. code-block:: python + + sitemap_meta = {"lastmod"} + + + def parse(self, response): + yield {"lastmod": response.meta.get("lastmod")} + + Use a dict to store an entry key under a different meta key, e.g. + ``sitemap_meta = {"lastmod": "sitemap_lastmod"}``. + + See :meth:`sitemap_filter` for the available entry keys. Keys missing + from an entry are skipped. + + By default no sitemap data is copied. + .. method:: sitemap_filter(entries) This is a filter function that could be overridden to select sitemap entries @@ -856,23 +880,26 @@ SitemapSpider if date_time.year >= 2005: yield entry - def parse(self, response): - # At the same time, you can get lastmod from meta - item = your_item() - item["lastmod"] = response.meta.get("lastmod") - This would retrieve only ``entries`` modified on 2005 and the following years. Entries are dict objects extracted from the sitemap document. Usually, the key is the tag name and the value is the text inside it. + The `sitemaps protocol`_ defines ``loc``, ``lastmod``, ``changefreq`` + and ``priority`` for ``urlset`` entries, and ``loc`` and ``lastmod`` + for ``sitemapindex`` entries. Sites may use additional tags, such as + those of the image, video and news sitemap extensions, which become + keys as well. + It's important to notice that: - as the loc attribute is required, entries without this tag are discarded - alternate links are stored in a list with the key ``alternate`` (see ``sitemap_alternate_links``) - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname`` + - the value is the text of the tag, so a tag with nested tags, such as + ````, gets an empty string If you omit this method, all entries found in sitemaps will be processed, observing other attributes and their settings. @@ -965,6 +992,7 @@ Combine SitemapSpider with other sources of urls: .. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/params.html .. _Sitemaps: https://www.sitemaps.org/index.html .. _Sitemap index files: https://www.sitemaps.org/protocol.html#index +.. _sitemaps protocol: https://www.sitemaps.org/protocol.html .. _robots.txt: https://www.robotstxt.org/ .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 6ae706315..4ddc5f21f 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -4,7 +4,7 @@ import logging import re # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation -from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 +from collections.abc import AsyncIterator, Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse @@ -30,6 +30,7 @@ class SitemapSpider(Spider): ] sitemap_follow: Sequence[re.Pattern[str] | str] = [""] sitemap_alternate_links: bool = False + sitemap_meta: Mapping[str, str] | Iterable[str] = () _max_size: int _warn_size: int @@ -52,6 +53,11 @@ class SitemapSpider(Spider): c = cast("CallbackT", getattr(self, c)) # noqa: PLW2901 self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] + self._meta_keys: list[tuple[str, str]] = list( + self.sitemap_meta.items() + if isinstance(self.sitemap_meta, Mapping) + else ((key, key) for key in self.sitemap_meta) + ) async def start(self) -> AsyncIterator[Any]: for url in self.sitemap_urls: @@ -69,8 +75,7 @@ class SitemapSpider(Spider): def _parse_sitemap(self, response: Response) -> Iterable[Request]: if response.url.endswith("/robots.txt"): urls = list(sitemap_urls_from_robots(response.body, base_url=response.url)) - yield from (Request(url, callback=self._parse_sitemap) for url in urls) - return + return (Request(url, callback=self._parse_sitemap) for url in urls) body = self._get_sitemap_body(response) if not body: @@ -79,28 +84,22 @@ class SitemapSpider(Spider): {"response": response}, extra={"spider": self}, ) - return + return () s = Sitemap(body) if s.type == "sitemapindex": urls = list(self._get_urls_from_sitemapindex(self.sitemap_filter(s))) - yield from (Request(loc, callback=self._parse_sitemap) for loc in urls) - return + return (Request(loc, callback=self._parse_sitemap) for loc in urls) if s.type == "urlset": - for entry in self.sitemap_filter(s): - meta = {"lastmod": entry.get("lastmod")} - if (loc := entry["loc"]) and (c := self._get_callback_from_url(loc)): - yield Request(loc, c, meta=meta) - # Also consider alternate URLs (xhtml:link rel="alternate") - if self.sitemap_alternate_links and ( - alt_list := entry.get("alternate") - ): - for loc in alt_list: - if c := self._get_callback_from_url(loc): - yield Request(loc, c, meta=meta) - return + url_callback_meta_triples = list( + self._get_urls_callbacks_and_meta_from_urlset(self.sitemap_filter(s)) + ) + return ( + Request(loc, callback=c, meta=meta) + for loc, c, meta in url_callback_meta_triples + ) logger.warning( "Ignoring invalid sitemap: %(response)s", @@ -108,6 +107,8 @@ class SitemapSpider(Spider): extra={"spider": self}, ) + return () + def _get_urls_from_sitemapindex( self, it: Iterable[dict[str, Any]] ) -> Iterable[str]: @@ -115,20 +116,20 @@ class SitemapSpider(Spider): if any(x.search(loc) for x in self._follow): yield loc - def _get_callback_from_url(self, url: str) -> CallbackT | None: - for r, c in self._cbs: - if r.search(url): - return c - return None - - def _get_urls_and_callbacks_from_urlset( + def _get_urls_callbacks_and_meta_from_urlset( self, it: Iterable[dict[str, Any]] - ) -> Iterable[tuple[str, CallbackT]]: - for loc in iterloc(it, self.sitemap_alternate_links): - for r, c in self._cbs: - if r.search(loc): - yield loc, c - break + ) -> Iterable[tuple[str, CallbackT, dict[str, Any] | None]]: + for entry in it: + meta = { + meta_key: entry[key] + for key, meta_key in self._meta_keys + if key in entry + } or None + for loc in iterloc((entry,), self.sitemap_alternate_links): + for r, c in self._cbs: + if r.search(loc): + yield loc, c, meta + break def _get_sitemap_body(self, response: Response) -> bytes | None: """Return the sitemap body contained in the given response, diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index 2f1ccab81..c4c7f432e 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -20,6 +20,8 @@ from tests.utils.crawl import crawl_items from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + from tests.mockserver.http import MockServer @@ -315,6 +317,75 @@ Sitemap: /sitemap-relative-url.xml items, _ = await crawl_items(_Spider, mockserver) assert items == [{"url": mockserver.url("/text")}] + @pytest.mark.parametrize( + ("entry_keys", "expected_meta"), + [ + ((), {}), + ( + {"lastmod", "changefreq"}, + {"lastmod": "2005-01-01", "changefreq": "daily"}, + ), + ({"lastmod": "sitemap_lastmod"}, {"sitemap_lastmod": "2005-01-01"}), + ], + ) + @coroutine_test + async def test_sitemap_meta( + self, + entry_keys: Mapping[str, str] | Iterable[str], + expected_meta: dict[str, str], + mockserver: MockServer, + ): + meta_keys = {"lastmod", "changefreq", "sitemap_lastmod"} + + class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc] + sitemap_meta = entry_keys + + def parse(self, response): + yield {k: v for k, v in response.meta.items() if k in meta_keys} + + def raw_body(self): + loc = self.mockserver.url("/text") + return ( + '' + '' + f"{loc}2005-01-01" + "daily0.8" + "" + ) + + items, _ = await crawl_items(_Spider, mockserver) + assert items == [expected_meta] + + @coroutine_test + async def test_sitemap_meta_alternate_links(self, mockserver: MockServer): + # Alternate links get the meta of the entry that declares them. + class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc] + sitemap_alternate_links = True + sitemap_meta = {"lastmod"} + + def parse(self, response): + yield {"url": response.url, "lastmod": response.meta.get("lastmod")} + + def raw_body(self): + loc = self.mockserver.url("/text") + alt = self.mockserver.url("/text?alt") + return ( + '' + '' + f"{loc}2005-01-01" + f'' + f"{self.mockserver.url('/text?no-lastmod')}" + "" + ) + + items, _ = await crawl_items(_Spider, mockserver) + assert sorted(items, key=lambda item: item["url"]) == [ + {"url": mockserver.url("/text"), "lastmod": "2005-01-01"}, + {"url": mockserver.url("/text?alt"), "lastmod": "2005-01-01"}, + {"url": mockserver.url("/text?no-lastmod"), "lastmod": None}, + ] + def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None: r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"") spider = self.spider_class("example.com")