also fetch alternate URLs from sitemaps, see #360

scrapy/contrib/spiders/sitemap.py

@@ -13,6 +13,10 @@ class SitemapSpider(BaseSpider):
     sitemap_follow = ['']
 
     def __init__(self, *a, **kw):
+        self._alternate = False
+        if 'alternate' in kw and kw.pop('alternate') == True:
+            self._alternate = True
+
         super(SitemapSpider, self).__init__(*a, **kw)
         self._cbs = []
         for r, c in self.sitemap_rules:
@@ -37,7 +41,7 @@ class SitemapSpider(BaseSpider):
 
             s = Sitemap(body)
             if s.type == 'sitemapindex':
-                for loc in iterloc(s):
+                for loc in iterloc(s, self._alternate):
                     if any(x.search(loc) for x in self._follow):
                         yield Request(loc, callback=self._parse_sitemap)
             elif s.type == 'urlset':
@@ -65,6 +69,11 @@ def regex(x):
         return re.compile(x)
     return x
 
-def iterloc(it):
+def iterloc(it, alt=False):
     for d in it:
         yield d['loc']
+
+        # Also consider alternate URLs (xhtml:link rel="alternate")
+        if alt == True and 'alternate' in d:
+            for l in d['alternate']:
+                yield l

scrapy/tests/test_utils_sitemap.py

@@ -159,6 +159,27 @@ Disallow: /forum/active/
             {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap3.xml'},
         ])
 
+    def test_alternate(self):
+        s = Sitemap("""<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xhtml="http://www.w3.org/1999/xhtml">
+        <url>
+            <loc>http://www.example.com/english/</loc>
+            <xhtml:link rel="alternate" hreflang="de"
+                href="http://www.example.com/deutsch/"/>
+            <xhtml:link rel="alternate" hreflang="de-ch"
+                href="http://www.example.com/schweiz-deutsch/"/>
+            <xhtml:link rel="alternate" hreflang="en"
+                href="http://www.example.com/english/"/>
+        </url>
+    </urlset>""")
+        
+        self.assertEqual(list(s), [
+            {'loc': 'http://www.example.com/english/',
+             'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
+            }
+        ])
+
 
 if __name__ == '__main__':
     unittest.main()

scrapy/utils/sitemap.py

@@ -23,7 +23,12 @@ class Sitemap(object):
             for el in elem.getchildren():
                 tag = el.tag
                 name = tag.split('}', 1)[1] if '}' in tag else tag
-                d[name] = el.text.strip() if el.text else ''
+
+                if name == 'link':
+                    d.setdefault('alternate', []).append(el.get('href'))
+                else:
+                    d[name] = el.text.strip() if el.text else ''
+            
             if 'loc' in d:
                 yield d