Merge pull request #1902 from starrify/case-insensitive-robots-txt-for-sitemap

[MRG+1] Added: Making it case-insensitive when extracting sitemap URLs from a robots.txt
2016-04-04 15:23:03 +02:00 · 2016-04-04 15:23:03 +02:00 · 642fedb3d6
parent bf7f675493 103f6eaa88
commit 642fedb3d6
1 changed files with 1 additions and 1 deletions
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@ -39,5 +39,5 @@ def sitemap_urls_from_robots(robots_text):
    robots.txt file
    """
    for line in robots_text.splitlines():
-        if line.lstrip().startswith('Sitemap:'):
+        if line.lstrip().lower().startswith('sitemap:'):
            yield line.split(':', 1)[1].strip()