Do not ignore sitemaps served with a query string

SitemapSpider._get_sitemap_body checked response.url.endswith('.xml'),
which fails for sitemap URLs carrying a query string
(e.g. '.../sitemap_products_8.xml?from=1&to=2'), causing such sitemaps to
be silently ignored when the response was not already an XmlResponse.

Compare the extension against the URL path (via urlparse) instead of the
full URL, so query strings no longer defeat the check.

Closes #6293

scrapy/spiders/sitemap.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import logging
 import re
+from urllib.parse import urlparse
 
 # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation
 from collections.abc import AsyncIterator, Iterable, Sequence  # noqa: TC003
@@ -145,7 +146,10 @@ class SitemapSpider(Spider):
         # without actually being a .xml.gz file in the first place,
         # merely XML gzip-compressed on the fly,
         # in other word, here, we have plain XML
-        if response.url.endswith(".xml") or response.url.endswith(".xml.gz"):
+        # Check the URL path only, so sitemaps served with a query string
+        # (e.g. ".../sitemap_products_8.xml?from=1&to=2") are not ignored.
+        url_path = urlparse(response.url).path
+        if url_path.endswith(".xml") or url_path.endswith(".xml.gz"):
             return response.body
         return None
 

tests/test_spider_sitemap.py

@@ -59,6 +59,14 @@ class TestSitemapSpider(TestSpider):
         r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY)
         self.assertSitemapBody(r, self.BODY)
 
+    def test_get_sitemap_body_xml_url_with_query_string(self):
+        # Sitemaps served with a query string must not be ignored (#6293).
+        r = TextResponse(
+            url="http://www.example.com/sitemap_products_8.xml?from=1&to=2",
+            body=self.BODY,
+        )
+        self.assertSitemapBody(r, self.BODY)
+
     def test_get_sitemap_body_xml_url_compressed(self):
         r = Response(
             url="http://www.example.com/sitemap.xml.gz",