From 160bcbacfc722f9db1b40394c1e02398ef343dd0 Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Sat, 2 Apr 2016 02:04:50 +0800 Subject: [PATCH] Added: Making it case-insensitive when extracting sitemap URLs from a robots.txt --- scrapy/utils/sitemap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index bbf37bc28..008196435 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -39,5 +39,5 @@ def sitemap_urls_from_robots(robots_text): robots.txt file """ for line in robots_text.splitlines(): - if line.lstrip().startswith('Sitemap:'): + if line.lstrip().lower().startswith('sitemap:'): yield line.split(':', 1)[1].strip()