diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 2a80b8d24..9a58b87cc 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -6,6 +6,7 @@ import re # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 from typing import TYPE_CHECKING, Any, cast +from urllib.parse import urlparse from scrapy.http import Request, Response, XmlResponse from scrapy.spiders import Spider @@ -145,7 +146,8 @@ class SitemapSpider(Spider): # without actually being a .xml.gz file in the first place, # merely XML gzip-compressed on the fly, # in other word, here, we have plain XML - if response.url.endswith(".xml") or response.url.endswith(".xml.gz"): + url_path = urlparse(response.url).path + if url_path.endswith((".xml", ".xml.gz")): return response.body return None diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index 0af99ab6d..1c9fa9971 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -59,6 +59,19 @@ class TestSitemapSpider(TestSpider): r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY) self.assertSitemapBody(r, self.BODY) + @pytest.mark.parametrize( + "url", + [ + "http://www.example.com/sitemap.xml?from=1&to=2", + "http://www.example.com/sitemap.xml?page=3", + "http://www.example.com/sitemap.xml.gz?from=1&to=2", + ], + ) + def test_get_sitemap_body_xml_url_with_query_params(self, url: str) -> None: + """Sitemaps whose URL has a query string are not ignored (issue #6293).""" + r = TextResponse(url=url, body=self.BODY) + self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_body_xml_url_compressed(self): r = Response( url="http://www.example.com/sitemap.xml.gz",