Detect sitemaps regardless of the response class

scrapy/spiders/sitemap.py

@@ -8,6 +8,7 @@ from collections.abc import AsyncIterator, Iterable, Sequence  # noqa: TC003
 from typing import TYPE_CHECKING, Any, cast
 
 from scrapy.http import Request, Response, XmlResponse
+from scrapy.responsetypes import responsetypes
 from scrapy.spiders import Spider
 from scrapy.utils._compression import _DecompressionMaxSizeExceeded
 from scrapy.utils.gz import gunzip, gzip_magic_number
@@ -120,7 +121,16 @@ class SitemapSpider(Spider):
         """Return the sitemap body contained in the given response,
         or None if the response is not a sitemap.
         """
-        if isinstance(response, XmlResponse):
+        # Download handlers other than the built-in ones may use their own
+        # response classes, so the content type, the URL and the body are taken
+        # into account as well.
+        if (
+            isinstance(response, XmlResponse)
+            or responsetypes.from_args(
+                headers=response.headers, url=response.url, body=response.body
+            )
+            is XmlResponse
+        ):
             return response.body
         if gzip_magic_number(response):
             uncompressed_size = len(response.body)

tests/test_spider_sitemap.py

@@ -67,6 +67,30 @@ class TestSitemapSpider(TestSpiderBase):
         r = XmlResponse(url="http://www.example.com/", body=b"")
         self.assertSitemapBody(r, b"")
 
+    def test_get_sitemap_body_xml_content_type(self):
+        r = TextResponse(
+            url="http://www.example.com/sitemap",
+            body=self.BODY,
+            headers={"Content-Type": "application/xml"},
+        )
+        self.assertSitemapBody(r, self.BODY)
+
+        r = TextResponse(
+            url="http://www.example.com/sitemap",
+            body=self.BODY,
+            headers={"Content-Type": "text/html"},
+        )
+        self.assertSitemapBody(r, None)
+
+    def test_get_sitemap_body_xml_body(self):
+        body = b'<?xml version="1.0" encoding="UTF-8"?><urlset></urlset>'
+        r = TextResponse(url="http://www.example.com/sitemap", body=body)
+        self.assertSitemapBody(r, body)
+
+    def test_get_sitemap_body_xml_file_url(self):
+        r = Response(url="file:///tmp/sitemap.rss", body=self.BODY)
+        self.assertSitemapBody(r, self.BODY)
+
     def test_get_sitemap_body_gzip_headers(self):
         r = Response(
             url="http://www.example.com/sitemap",