diff --git a/tests/test_spider.py b/tests/test_spider.py index f26da2334..871852ab2 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -375,6 +375,38 @@ Sitemap: /sitemap-relative-url.xml 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/italiano/']) + def test_sitemap_filter(self): + sitemap = b""" + + + http://www.example.com/english/ + 2010-01-01 + + + http://www.example.com/portuguese/ + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, urls): + from datetime import datetime + for url in urls: + date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + if date_time.year > 2008: + yield url + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/', + 'http://www.example.com/portuguese/']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/']) + class DeprecationTest(unittest.TestCase):