RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SitemapSpider: ignore non-xml responses. fixes #331

This commit is contained in:
Pablo Hoffman 2011-06-27 10:02:16 -03:00
parent db5cae7c03
commit 7cd559eca5
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/contrib/spiders/sitemap.py
View File

@ -1,8 +1,9 @@
import re
from scrapy.spider import BaseSpider
from scrapy.http import Request
from scrapy.http import Request, XmlResponse
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
from scrapy import log
class SitemapSpider(BaseSpider):
@ -27,6 +28,9 @@ class SitemapSpider(BaseSpider):
for url in sitemap_urls_from_robots(response.body):
yield Request(url, callback=self._parse_sitemap)
else:
if not isinstance(response, XmlResponse):
log.msg("Ignoring non-XML sitemap: %s" % response, log.WARNING)
return
s = Sitemap(response.body)
if s.type == 'sitemapindex':
for loc in iterloc(s):