From 7cd559eca54157d1bf37af0f651abfaaf22cdd5f Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 27 Jun 2011 10:02:16 -0300 Subject: [PATCH] SitemapSpider: ignore non-xml responses. fixes #331 --- scrapy/contrib/spiders/sitemap.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py index 6cee49fb3..6b694ad4d 100644 --- a/scrapy/contrib/spiders/sitemap.py +++ b/scrapy/contrib/spiders/sitemap.py @@ -1,8 +1,9 @@ import re from scrapy.spider import BaseSpider -from scrapy.http import Request +from scrapy.http import Request, XmlResponse from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots +from scrapy import log class SitemapSpider(BaseSpider): @@ -27,6 +28,9 @@ class SitemapSpider(BaseSpider): for url in sitemap_urls_from_robots(response.body): yield Request(url, callback=self._parse_sitemap) else: + if not isinstance(response, XmlResponse): + log.msg("Ignoring non-XML sitemap: %s" % response, log.WARNING) + return s = Sitemap(response.body) if s.type == 'sitemapindex': for loc in iterloc(s):