From a19dfafb25d0651eaf254471ef4b2c59658b06a9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 16 Apr 2014 22:33:30 +0200 Subject: [PATCH] Support case-insensitive domains in url_is_from_any_domain() --- scrapy/tests/test_utils_url.py | 4 ++++ scrapy/utils/url.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index a47bb883d..959760068 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -17,6 +17,10 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk'])) self.assertFalse(url_is_from_any_domain(url, ['art.co.uk'])) + url = 'http://www.Wheele-Bin-Art.co.uk/get/product/123' + self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.CO.UK'])) + self.assertTrue(url_is_from_any_domain(url, ['WHEELE-BIN-ART.CO.UK'])) + url = 'http://192.169.0.15:8080/mypage.html' self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080'])) self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15'])) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 918042064..9a12aa26b 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -17,10 +17,10 @@ from scrapy.utils.python import unicode_to_str def url_is_from_any_domain(url, domains): """Return True if the url belongs to any of the given domains""" - host = parse_url(url).netloc + host = parse_url(url).netloc.lower() if host: - return any(((host == d) or (host.endswith('.%s' % d)) for d in domains)) + return any(((host == d.lower()) or (host.endswith('.%s' % d.lower())) for d in domains)) else: return False