RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use a spider attribute to allow enabling sitemap metadata

This commit is contained in:
Adrian Chaves 2026-08-03 14:32:45 +02:00
parent fb46b5b7c3
commit aa57a07a82
3 changed files with 136 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
docs/topics/spiders.rst
View File

@ -823,6 +823,30 @@ SitemapSpider
Default is ``sitemap_alternate_links`` disabled.
.. attribute:: sitemap_meta
Sitemap entry keys to copy into the :attr:`~scrapy.Request.meta` of the
requests generated from those entries, so that callbacks can read
sitemap data such as ``lastmod``:
.. versionadded:: VERSION
.. code-block:: python
sitemap_meta = {"lastmod"}
def parse(self, response):
yield {"lastmod": response.meta.get("lastmod")}
Use a dict to store an entry key under a different meta key, e.g.
``sitemap_meta = {"lastmod": "sitemap_lastmod"}``.
See :meth:`sitemap_filter` for the available entry keys. Keys missing
from an entry are skipped.
By default no sitemap data is copied.
.. method:: sitemap_filter(entries)
This is a filter function that could be overridden to select sitemap entries
@ -856,23 +880,26 @@ SitemapSpider
if date_time.year >= 2005:
yield entry
def parse(self, response):
# At the same time, you can get lastmod from meta
item = your_item()
item["lastmod"] = response.meta.get("lastmod")
This would retrieve only ``entries`` modified on 2005 and the following
years.
Entries are dict objects extracted from the sitemap document.
Usually, the key is the tag name and the value is the text inside it.
The `sitemaps protocol`_ defines ``loc``, ``lastmod``, ``changefreq``
and ``priority`` for ``urlset`` entries, and ``loc`` and ``lastmod``
for ``sitemapindex`` entries. Sites may use additional tags, such as
those of the image, video and news sitemap extensions, which become
keys as well.
It's important to notice that:
- as the loc attribute is required, entries without this tag are discarded
- alternate links are stored in a list with the key ``alternate``
(see ``sitemap_alternate_links``)
- namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname``
- the value is the text of the tag, so a tag with nested tags, such as
``<image:image>``, gets an empty string
If you omit this method, all entries found in sitemaps will be
processed, observing other attributes and their settings.
@ -965,6 +992,7 @@ Combine SitemapSpider with other sources of urls:
.. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/params.html
.. _Sitemaps: https://www.sitemaps.org/index.html
.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index
.. _sitemaps protocol: https://www.sitemaps.org/protocol.html
.. _robots.txt: https://www.robotstxt.org/
.. _TLD: https://en.wikipedia.org/wiki/Top-level_domain
.. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/

63
scrapy/spiders/sitemap.py
View File

@ -4,7 +4,7 @@ import logging
import re
# Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation
from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
from typing import TYPE_CHECKING, Any, cast
from scrapy.http import Request, Response, XmlResponse
@ -30,6 +30,7 @@ class SitemapSpider(Spider):
]
sitemap_follow: Sequence[re.Pattern[str] | str] = [""]
sitemap_alternate_links: bool = False
sitemap_meta: Mapping[str, str] | Iterable[str] = ()
_max_size: int
_warn_size: int
@ -52,6 +53,11 @@ class SitemapSpider(Spider):
c = cast("CallbackT", getattr(self, c)) # noqa: PLW2901
self._cbs.append((regex(r), c))
self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]
self._meta_keys: list[tuple[str, str]] = list(
self.sitemap_meta.items()
if isinstance(self.sitemap_meta, Mapping)
else ((key, key) for key in self.sitemap_meta)
)
async def start(self) -> AsyncIterator[Any]:
for url in self.sitemap_urls:
@ -69,8 +75,7 @@ class SitemapSpider(Spider):
def _parse_sitemap(self, response: Response) -> Iterable[Request]:
if response.url.endswith("/robots.txt"):
urls = list(sitemap_urls_from_robots(response.body, base_url=response.url))
yield from (Request(url, callback=self._parse_sitemap) for url in urls)
return
return (Request(url, callback=self._parse_sitemap) for url in urls)
body = self._get_sitemap_body(response)
if not body:
@ -79,28 +84,22 @@ class SitemapSpider(Spider):
{"response": response},
extra={"spider": self},
)
return
return ()
s = Sitemap(body)
if s.type == "sitemapindex":
urls = list(self._get_urls_from_sitemapindex(self.sitemap_filter(s)))
yield from (Request(loc, callback=self._parse_sitemap) for loc in urls)
return
return (Request(loc, callback=self._parse_sitemap) for loc in urls)
if s.type == "urlset":
for entry in self.sitemap_filter(s):
meta = {"lastmod": entry.get("lastmod")}
if (loc := entry["loc"]) and (c := self._get_callback_from_url(loc)):
yield Request(loc, c, meta=meta)
# Also consider alternate URLs (xhtml:link rel="alternate")
if self.sitemap_alternate_links and (
alt_list := entry.get("alternate")
):
for loc in alt_list:
if c := self._get_callback_from_url(loc):
yield Request(loc, c, meta=meta)
return
url_callback_meta_triples = list(
self._get_urls_callbacks_and_meta_from_urlset(self.sitemap_filter(s))
)
return (
Request(loc, callback=c, meta=meta)
for loc, c, meta in url_callback_meta_triples
)
logger.warning(
"Ignoring invalid sitemap: %(response)s",
@ -108,6 +107,8 @@ class SitemapSpider(Spider):
extra={"spider": self},
)
return ()
def _get_urls_from_sitemapindex(
self, it: Iterable[dict[str, Any]]
) -> Iterable[str]:
@ -115,20 +116,20 @@ class SitemapSpider(Spider):
if any(x.search(loc) for x in self._follow):
yield loc
def _get_callback_from_url(self, url: str) -> CallbackT | None:
for r, c in self._cbs:
if r.search(url):
return c
return None
def _get_urls_and_callbacks_from_urlset(
def _get_urls_callbacks_and_meta_from_urlset(
self, it: Iterable[dict[str, Any]]
) -> Iterable[tuple[str, CallbackT]]:
for loc in iterloc(it, self.sitemap_alternate_links):
for r, c in self._cbs:
if r.search(loc):
yield loc, c
break
) -> Iterable[tuple[str, CallbackT, dict[str, Any] | None]]:
for entry in it:
meta = {
meta_key: entry[key]
for key, meta_key in self._meta_keys
if key in entry
} or None
for loc in iterloc((entry,), self.sitemap_alternate_links):
for r, c in self._cbs:
if r.search(loc):
yield loc, c, meta
break
def _get_sitemap_body(self, response: Response) -> bytes | None:
"""Return the sitemap body contained in the given response,

71
tests/test_spider_sitemap.py
View File

@ -20,6 +20,8 @@ from tests.utils.crawl import crawl_items
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import Iterable, Mapping
from tests.mockserver.http import MockServer
@ -315,6 +317,75 @@ Sitemap: /sitemap-relative-url.xml
items, _ = await crawl_items(_Spider, mockserver)
assert items == [{"url": mockserver.url("/text")}]
@pytest.mark.parametrize(
("entry_keys", "expected_meta"),
[
((), {}),
(
{"lastmod", "changefreq"},
{"lastmod": "2005-01-01", "changefreq": "daily"},
),
({"lastmod": "sitemap_lastmod"}, {"sitemap_lastmod": "2005-01-01"}),
],
)
@coroutine_test
async def test_sitemap_meta(
self,
entry_keys: Mapping[str, str] | Iterable[str],
expected_meta: dict[str, str],
mockserver: MockServer,
):
meta_keys = {"lastmod", "changefreq", "sitemap_lastmod"}
class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc]
sitemap_meta = entry_keys
def parse(self, response):
yield {k: v for k, v in response.meta.items() if k in meta_keys}
def raw_body(self):
loc = self.mockserver.url("/text")
return (
'<?xml version="1.0" encoding="UTF-8"?>'
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
f"<url><loc>{loc}</loc><lastmod>2005-01-01</lastmod>"
"<changefreq>daily</changefreq><priority>0.8</priority></url>"
"</urlset>"
)
items, _ = await crawl_items(_Spider, mockserver)
assert items == [expected_meta]
@coroutine_test
async def test_sitemap_meta_alternate_links(self, mockserver: MockServer):
# Alternate links get the meta of the entry that declares them.
class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc]
sitemap_alternate_links = True
sitemap_meta = {"lastmod"}
def parse(self, response):
yield {"url": response.url, "lastmod": response.meta.get("lastmod")}
def raw_body(self):
loc = self.mockserver.url("/text")
alt = self.mockserver.url("/text?alt")
return (
'<?xml version="1.0" encoding="UTF-8"?>'
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"'
' xmlns:xhtml="http://www.w3.org/1999/xhtml">'
f"<url><loc>{loc}</loc><lastmod>2005-01-01</lastmod>"
f'<xhtml:link rel="alternate" hreflang="de" href="{alt}"/></url>'
f"<url><loc>{self.mockserver.url('/text?no-lastmod')}</loc></url>"
"</urlset>"
)
items, _ = await crawl_items(_Spider, mockserver)
assert sorted(items, key=lambda item: item["url"]) == [
{"url": mockserver.url("/text"), "lastmod": "2005-01-01"},
{"url": mockserver.url("/text?alt"), "lastmod": "2005-01-01"},
{"url": mockserver.url("/text?no-lastmod"), "lastmod": None},
]
def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None:
r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")
spider = self.spider_class("example.com")