diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 6586db668..82e96df1e 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -560,6 +560,25 @@ SitemapSpider
By default, all sitemaps are followed.
+ .. attribute:: sitemap_alternate_links
+
+ Specifies if alternate links for one ``url`` should be followed. These
+ are links for the same website in another language passed within
+ the same ``url`` block.
+
+ For example::
+
+
+ http://example.com/
+
+
+
+ With ``sitemap_alternate_links`` set, this would retrieve both URLs. With
+ ``sitemap_alternate_links`` disabled, only ``http://example.com/`` would be
+ retrieved.
+
+ Default is ``sitemap_alternate_links`` disabled.
+
SitemapSpider examples
~~~~~~~~~~~~~~~~~~~~~~
diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py
index 4fc19e108..d8567f7f0 100644
--- a/scrapy/contrib/spiders/sitemap.py
+++ b/scrapy/contrib/spiders/sitemap.py
@@ -11,6 +11,7 @@ class SitemapSpider(BaseSpider):
sitemap_urls = ()
sitemap_rules = [('', 'parse')]
sitemap_follow = ['']
+ sitemap_alternate_links = False
def __init__(self, *a, **kw):
super(SitemapSpider, self).__init__(*a, **kw)
@@ -37,7 +38,7 @@ class SitemapSpider(BaseSpider):
s = Sitemap(body)
if s.type == 'sitemapindex':
- for loc in iterloc(s):
+ for loc in iterloc(s, self.sitemap_alternate_links):
if any(x.search(loc) for x in self._follow):
yield Request(loc, callback=self._parse_sitemap)
elif s.type == 'urlset':
@@ -65,6 +66,11 @@ def regex(x):
return re.compile(x)
return x
-def iterloc(it):
+def iterloc(it, alt=False):
for d in it:
yield d['loc']
+
+ # Also consider alternate URLs (xhtml:link rel="alternate")
+ if alt and 'alternate' in d:
+ for l in d['alternate']:
+ yield l
diff --git a/scrapy/tests/test_utils_sitemap.py b/scrapy/tests/test_utils_sitemap.py
index 8be94a309..0049e5c40 100644
--- a/scrapy/tests/test_utils_sitemap.py
+++ b/scrapy/tests/test_utils_sitemap.py
@@ -159,6 +159,42 @@ Disallow: /forum/active/
{'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap3.xml'},
])
+ def test_comment(self):
+ s = Sitemap("""
+
+
+ http://www.example.com/
+
+
+ """)
+
+ self.assertEqual(list(s), [
+ {'loc': 'http://www.example.com/'}
+ ])
+
+ def test_alternate(self):
+ s = Sitemap("""
+
+
+ http://www.example.com/english/
+
+
+
+
+
+ """)
+
+ self.assertEqual(list(s), [
+ {'loc': 'http://www.example.com/english/',
+ 'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
+ }
+ ])
+
if __name__ == '__main__':
unittest.main()
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 38e38d6a9..24a540531 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -12,7 +12,7 @@ class Sitemap(object):
(type=sitemapindex) files"""
def __init__(self, xmltext):
- xmlp = lxml.etree.XMLParser(recover=True)
+ xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True)
self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
rt = self._root.tag
self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt
@@ -23,7 +23,13 @@ class Sitemap(object):
for el in elem.getchildren():
tag = el.tag
name = tag.split('}', 1)[1] if '}' in tag else tag
- d[name] = el.text.strip() if el.text else ''
+
+ if name == 'link':
+ if 'href' in el.attrib:
+ d.setdefault('alternate', []).append(el.get('href'))
+ else:
+ d[name] = el.text.strip() if el.text else ''
+
if 'loc' in d:
yield d