SitemapSpider: added support for filtering which sitemaps to follow (patch contributed by Rolando Espinoza). closes #330

docs/topics/spiders.rst

@@ -512,6 +512,15 @@ SitemapSpider
         If you omit this attribute, all urls found in sitemaps will be
         processed with the ``parse`` callback.
 
+    .. attribute:: sitemap_follow
+
+        A list of regexes of sitemap that should be followed. This is is only
+        for sites that use `Sitemap index files`_ that point to other sitemap
+        files.
+
+        By default, all sitemaps are followed.
+
+
 SitemapSpider examples
 ~~~~~~~~~~~~~~~~~~~~~~
 
@@ -544,7 +553,8 @@ callback::
         def parse_category(self, response):
             pass # ... scrape category ...
 
-Follow sitemaps defined in the `robots.txt`_ file::
+Follow sitemaps defined in the `robots.txt`_ file and only follow sitemaps
+whose url contains ``/sitemap_shop``::
 
     from scrapy.contrib.spiders import SitemapSpider
 
@@ -553,6 +563,7 @@ Follow sitemaps defined in the `robots.txt`_ file::
         sitemap_rules = [
             ('/shop/', 'parse_shop'),
         ]
+        sitemap_follow = ['/sitemap_shops']
 
         def parse_shop(self, response):
             pass # ... scrape shop here ...
@@ -581,5 +592,5 @@ Combine SitemapSpider with other sources of urls::
             pass # ... scrape other here ...
 
 .. _Sitemaps: http://www.sitemaps.org
+.. _Sitemap index files: http://www.sitemaps.org/protocol.php#index
 .. _robots.txt: http://www.robotstxt.org/
-

scrapy/contrib/spiders/sitemap.py

@@ -8,16 +8,16 @@ class SitemapSpider(BaseSpider):
 
     sitemap_urls = ()
     sitemap_rules = [('', 'parse')]
+    sitemap_follow = ['']
 
     def __init__(self, *a, **kw):
         super(SitemapSpider, self).__init__(*a, **kw)
         self._cbs = []
         for r, c in self.sitemap_rules:
-            if isinstance(r, basestring):
-                r = re.compile(r)
             if isinstance(c, basestring):
                 c = getattr(self, c)
-            self._cbs.append((r, c))
+            self._cbs.append((regex(r), c))
+        self._follow = [regex(x) for x in self.sitemap_follow]
 
     def start_requests(self):
         return [Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls]
@@ -29,12 +29,22 @@ class SitemapSpider(BaseSpider):
         else:
             s = Sitemap(response.body)
             if s.type == 'sitemapindex':
-                for sitemap in s:
-                    yield Request(sitemap['loc'], callback=self._parse_sitemap)
+                for loc in iterloc(s):
+                    if any(x.search(loc) for x in self._follow):
+                        yield Request(loc, callback=self._parse_sitemap)
             elif s.type == 'urlset':
-                for url in s:
-                    loc = url['loc']
+                for loc in iterloc(s):
                     for r, c in self._cbs:
                         if r.search(loc):
                             yield Request(loc, callback=c)
                             break
+
+
+def regex(x):
+    if isinstance(x, basestring):
+        return re.compile(x)
+    return x
+
+def iterloc(it):
+    for d in it:
+        yield d['loc']