diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 2a80b8d24..0f2a2edb3 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging import re +from urllib.parse import urlparse # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 @@ -145,7 +146,10 @@ class SitemapSpider(Spider): # without actually being a .xml.gz file in the first place, # merely XML gzip-compressed on the fly, # in other word, here, we have plain XML - if response.url.endswith(".xml") or response.url.endswith(".xml.gz"): + # Check the URL path only, so sitemaps served with a query string + # (e.g. ".../sitemap_products_8.xml?from=1&to=2") are not ignored. + url_path = urlparse(response.url).path + if url_path.endswith(".xml") or url_path.endswith(".xml.gz"): return response.body return None diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index 0af99ab6d..03d83734a 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -59,6 +59,14 @@ class TestSitemapSpider(TestSpider): r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY) self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_body_xml_url_with_query_string(self): + # Sitemaps served with a query string must not be ignored (#6293). + r = TextResponse( + url="http://www.example.com/sitemap_products_8.xml?from=1&to=2", + body=self.BODY, + ) + self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_body_xml_url_compressed(self): r = Response( url="http://www.example.com/sitemap.xml.gz",