diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a08dc30f2..b0b9e0483 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,6 +680,32 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. + .. attribute:: sitemap_filter + + Specifies a function to filter sitemap entries and their attributes. + + For example:: + + + http://example.com/ + 2005-01-01 + + + We can define a ``sitemap_filter`` function to filter ``urls`` by date:: + + def sitemap_filter(urls): + from datetime import datetime + for url in urls: + date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + if date_time.year >= 2005: + yield url + + This would retrieve only ``urls`` modified on 2005 and the following + years. + + If you omit this attribute, all urls found in sitemaps will be + processed, observing other attributes and their settings. + SitemapSpider examples ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 0ee8ba5e7..907aba243 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -17,6 +17,7 @@ class SitemapSpider(Spider): sitemap_rules = [('', 'parse')] sitemap_follow = [''] sitemap_alternate_links = False + sitemap_filter = None def __init__(self, *a, **kw): super(SitemapSpider, self).__init__(*a, **kw) @@ -43,12 +44,17 @@ class SitemapSpider(Spider): return s = Sitemap(body) + if callable(self.sitemap_filter): + it = self.sitemap_filter(s) + else: + it = s + if s.type == 'sitemapindex': - for loc in iterloc(s, self.sitemap_alternate_links): + for loc in iterloc(it, self.sitemap_alternate_links): if any(x.search(loc) for x in self._follow): yield Request(loc, callback=self._parse_sitemap) elif s.type == 'urlset': - for loc in iterloc(s, self.sitemap_alternate_links): + for loc in iterloc(it, self.sitemap_alternate_links): for r, c in self._cbs: if r.search(loc): yield Request(loc, callback=c)