diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index b0b9e0483..127c8d03e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,7 +680,7 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. - .. attribute:: sitemap_filter + .. method:: sitemap_filter(urls) Specifies a function to filter sitemap entries and their attributes. @@ -703,7 +703,7 @@ SitemapSpider This would retrieve only ``urls`` modified on 2005 and the following years. - If you omit this attribute, all urls found in sitemaps will be + If you omit this method, all urls found in sitemaps will be processed, observing other attributes and their settings. diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 907aba243..c86e986db 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -17,7 +17,6 @@ class SitemapSpider(Spider): sitemap_rules = [('', 'parse')] sitemap_follow = [''] sitemap_alternate_links = False - sitemap_filter = None def __init__(self, *a, **kw): super(SitemapSpider, self).__init__(*a, **kw) @@ -32,6 +31,14 @@ class SitemapSpider(Spider): for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) + def sitemap_filter(self, urls): + """This method can be used to filter sitemap entries by their + attributes, for example, you can filter locs with lastmod greater + than a given date (see docs). + """ + for url in urls: + yield url + def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): for url in sitemap_urls_from_robots(response.text, base_url=response.url): @@ -44,10 +51,7 @@ class SitemapSpider(Spider): return s = Sitemap(body) - if callable(self.sitemap_filter): - it = self.sitemap_filter(s) - else: - it = s + it = self.sitemap_filter(s) if s.type == 'sitemapindex': for loc in iterloc(it, self.sitemap_alternate_links):