Added: Making it case-insensitive when extracting sitemap URLs from a robots.txt

2016-04-02 02:04:50 +08:00 · 2016-04-02 02:04:50 +08:00 · 160bcbacfc
parent b2e914943d
commit 160bcbacfc
1 changed files with 1 additions and 1 deletions
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@ -39,5 +39,5 @@ def sitemap_urls_from_robots(robots_text):
    robots.txt file
    """
    for line in robots_text.splitlines():
-        if line.lstrip().startswith('Sitemap:'):
+        if line.lstrip().lower().startswith('sitemap:'):
            yield line.split(':', 1)[1].strip()