add sitemap_filter attribute to SitemapSpider class

it makes it possible to filter sitemap urls by any available attribute

for example, you can filter urls with lastmod greater than a given datetime

it can be helpful when the url loc itself does not aggregate that information

docs/topics/spiders.rst

@@ -680,6 +680,32 @@ SitemapSpider
 
         Default is ``sitemap_alternate_links`` disabled.
 
+    .. attribute:: sitemap_filter
+
+        Specifies a function to filter sitemap entries and their attributes.
+
+        For example::
+
+            <url>
+                <loc>http://example.com/</loc>
+                <lastmod>2005-01-01</lastmod>
+            </url>
+
+        We can define a ``sitemap_filter`` function to filter ``urls`` by date::
+
+            def sitemap_filter(urls):
+                from datetime import datetime
+                for url in urls:
+                    date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
+                    if date_time.year >= 2005:
+                        yield url
+
+        This would retrieve only ``urls`` modified on 2005 and the following
+        years.
+
+        If you omit this attribute, all urls found in sitemaps will be
+        processed, observing other attributes and their settings.
+
 
 SitemapSpider examples
 ~~~~~~~~~~~~~~~~~~~~~~

scrapy/spiders/sitemap.py

@@ -17,6 +17,7 @@ class SitemapSpider(Spider):
     sitemap_rules = [('', 'parse')]
     sitemap_follow = ['']
     sitemap_alternate_links = False
+    sitemap_filter = None
 
     def __init__(self, *a, **kw):
         super(SitemapSpider, self).__init__(*a, **kw)
@@ -43,12 +44,17 @@ class SitemapSpider(Spider):
                 return
 
             s = Sitemap(body)
+            if callable(self.sitemap_filter):
+                it = self.sitemap_filter(s)
+            else:
+                it = s
+
             if s.type == 'sitemapindex':
-                for loc in iterloc(s, self.sitemap_alternate_links):
+                for loc in iterloc(it, self.sitemap_alternate_links):
                     if any(x.search(loc) for x in self._follow):
                         yield Request(loc, callback=self._parse_sitemap)
             elif s.type == 'urlset':
-                for loc in iterloc(s, self.sitemap_alternate_links):
+                for loc in iterloc(it, self.sitemap_alternate_links):
                     for r, c in self._cbs:
                         if r.search(loc):
                             yield Request(loc, callback=c)