From 25c56159b86288311630cc0cf6db9d755aeeff1e Mon Sep 17 00:00:00 2001 From: orangain Date: Sat, 6 Feb 2016 22:26:46 +0900 Subject: [PATCH] Fix SitemapSpider to extract sitemap urls from robots.txt properly This will fix #1766. --- scrapy/spiders/sitemap.py | 2 +- tests/test_spider.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index eede467a8..89d96c330 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -32,7 +32,7 @@ class SitemapSpider(Spider): def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): - for url in sitemap_urls_from_robots(response.body): + for url in sitemap_urls_from_robots(response.text): yield Request(url, callback=self._parse_sitemap) else: body = self._get_sitemap_body(response) diff --git a/tests/test_spider.py b/tests/test_spider.py index 4d5d4b07e..1d22c1212 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -328,6 +328,18 @@ class SitemapSpiderTest(SpiderTest): r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_urls_from_robotstxt(self): + robots = b"""# Sitemap files +Sitemap: http://example.com/sitemap.xml +Sitemap: http://example.com/sitemap-product-index.xml +""" + + r = TextResponse(url="http://www.example.com/robots.txt", body=robots) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://example.com/sitemap.xml', + 'http://example.com/sitemap-product-index.xml']) + class BaseSpiderDeprecationTest(unittest.TestCase):