RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2854 from jenya/2853-sitemap-follow-alternate-fix 

[MRG+1] Follow alternate link for all types of sitemaps #2853
This commit is contained in:
Mikhail Korobov 2017-08-21 19:15:03 +05:00 committed by GitHub
parent 984c0c19ee 0cb3085f84
commit 885289f497
2 changed files with 28 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/spiders/sitemap.py
View File

@ -48,7 +48,7 @@ class SitemapSpider(Spider):
if any(x.search(loc) for x in self._follow):
yield Request(loc, callback=self._parse_sitemap)
elif s.type == 'urlset':
for loc in iterloc(s):
for loc in iterloc(s, self.sitemap_alternate_links):
for r, c in self._cbs:
if r.search(loc):
yield Request(loc, callback=c)

27
tests/test_spider.py
View File

@ -348,6 +348,33 @@ Sitemap: /sitemap-relative-url.xml
'http://example.com/sitemap-uppercase.xml',
'http://www.example.com/sitemap-relative-url.xml'])
def test_alternate_url_locs(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>http://www.example.com/english/</loc>
<xhtml:link rel="alternate" hreflang="de"
href="http://www.example.com/deutsch/"/>
<xhtml:link rel="alternate" hreflang="de-ch"
href="http://www.example.com/schweiz-deutsch/"/>
<xhtml:link rel="alternate" hreflang="it"
href="http://www.example.com/italiano/"/>
<xhtml:link rel="alternate" hreflang="it"/><!-- wrong tag without href -->
</url>
</urlset>"""
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
self.assertEqual([req.url for req in spider._parse_sitemap(r)],
['http://www.example.com/english/'])
spider.sitemap_alternate_links = True
self.assertEqual([req.url for req in spider._parse_sitemap(r)],
['http://www.example.com/english/',
'http://www.example.com/deutsch/',
'http://www.example.com/schweiz-deutsch/',
'http://www.example.com/italiano/'])
class DeprecationTest(unittest.TestCase):