diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index a47bb883d..959760068 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -17,6 +17,10 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk'])) self.assertFalse(url_is_from_any_domain(url, ['art.co.uk'])) + url = 'http://www.Wheele-Bin-Art.co.uk/get/product/123' + self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.CO.UK'])) + self.assertTrue(url_is_from_any_domain(url, ['WHEELE-BIN-ART.CO.UK'])) + url = 'http://192.169.0.15:8080/mypage.html' self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080'])) self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15'])) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 918042064..9a12aa26b 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -17,10 +17,10 @@ from scrapy.utils.python import unicode_to_str def url_is_from_any_domain(url, domains): """Return True if the url belongs to any of the given domains""" - host = parse_url(url).netloc + host = parse_url(url).netloc.lower() if host: - return any(((host == d) or (host.endswith('.%s' % d)) for d in domains)) + return any(((host == d.lower()) or (host.endswith('.%s' % d.lower())) for d in domains)) else: return False