diff --git a/tests/test_spider.py b/tests/test_spider.py index 8b56cfec1..fefdaa403 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -407,6 +407,41 @@ Sitemap: /sitemap-relative-url.xml self.assertEqual([req.url for req in spider._parse_sitemap(r)], ['http://www.example.com/english/']) + def test_sitemap_filter_with_alternate_links(self): + sitemap = b""" + + + http://www.example.com/english/article_1/ + 2010-01-01 + + + + http://www.example.com/english/article_2/ + 2015-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + alternate_links = entry.get('alternate', tuple()) + for link in alternate_links: + if '/deutsch/' in link: + entry['loc'] = link + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/article_1/', + 'http://www.example.com/english/article_2/']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/deutsch/article_1/']) + def test_sitemapindex_filter(self): sitemap = b"""