diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py index 4fc19e108..e4092c5d6 100644 --- a/scrapy/contrib/spiders/sitemap.py +++ b/scrapy/contrib/spiders/sitemap.py @@ -13,6 +13,10 @@ class SitemapSpider(BaseSpider): sitemap_follow = [''] def __init__(self, *a, **kw): + self._alternate = False + if 'alternate' in kw and kw.pop('alternate') == True: + self._alternate = True + super(SitemapSpider, self).__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: @@ -37,7 +41,7 @@ class SitemapSpider(BaseSpider): s = Sitemap(body) if s.type == 'sitemapindex': - for loc in iterloc(s): + for loc in iterloc(s, self._alternate): if any(x.search(loc) for x in self._follow): yield Request(loc, callback=self._parse_sitemap) elif s.type == 'urlset': @@ -65,6 +69,11 @@ def regex(x): return re.compile(x) return x -def iterloc(it): +def iterloc(it, alt=False): for d in it: yield d['loc'] + + # Also consider alternate URLs (xhtml:link rel="alternate") + if alt == True and 'alternate' in d: + for l in d['alternate']: + yield l diff --git a/scrapy/tests/test_utils_sitemap.py b/scrapy/tests/test_utils_sitemap.py index 8be94a309..a338adfe2 100644 --- a/scrapy/tests/test_utils_sitemap.py +++ b/scrapy/tests/test_utils_sitemap.py @@ -159,6 +159,27 @@ Disallow: /forum/active/ {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap3.xml'}, ]) + def test_alternate(self): + s = Sitemap(""" + + + http://www.example.com/english/ + + + + + """) + + self.assertEqual(list(s), [ + {'loc': 'http://www.example.com/english/', + 'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/'] + } + ]) + if __name__ == '__main__': unittest.main() diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 38e38d6a9..b69b4226f 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -23,7 +23,12 @@ class Sitemap(object): for el in elem.getchildren(): tag = el.tag name = tag.split('}', 1)[1] if '}' in tag else tag - d[name] = el.text.strip() if el.text else '' + + if name == 'link': + d.setdefault('alternate', []).append(el.get('href')) + else: + d[name] = el.text.strip() if el.text else '' + if 'loc' in d: yield d