RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1767 from orangain/sitemap-robotstxt 

[MRG+1] PY3: Fix SitemapSpider to extract sitemap urls from robots.txt properly
This commit is contained in:
Mikhail Korobov 2016-02-08 10:11:58 +05:00
parent e328a9b9df 25c56159b8
commit 44bc4c06ff
2 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/spiders/sitemap.py
View File

@ -32,7 +32,7 @@ class SitemapSpider(Spider):
def _parse_sitemap(self, response):
if response.url.endswith('/robots.txt'):
for url in sitemap_urls_from_robots(response.body):
for url in sitemap_urls_from_robots(response.text):
yield Request(url, callback=self._parse_sitemap)
else:
body = self._get_sitemap_body(response)

12
tests/test_spider.py
View File

@ -328,6 +328,18 @@ class SitemapSpiderTest(SpiderTest):
r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY)
self.assertSitemapBody(r, self.BODY)
def test_get_sitemap_urls_from_robotstxt(self):
robots = b"""# Sitemap files
Sitemap: http://example.com/sitemap.xml
Sitemap: http://example.com/sitemap-product-index.xml
"""
r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
spider = self.spider_class("example.com")
self.assertEqual([req.url for req in spider._parse_sitemap(r)],
['http://example.com/sitemap.xml',
'http://example.com/sitemap-product-index.xml'])
class BaseSpiderDeprecationTest(unittest.TestCase):