RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support case-insensitive domains in url_is_from_any_domain()

This commit is contained in:
Paul Tremberth 2014-04-16 22:33:30 +02:00
parent 7814cbf319
commit a19dfafb25
2 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/tests/test_utils_url.py
View File

@ -17,6 +17,10 @@ class UrlUtilsTest(unittest.TestCase):
self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk']))
self.assertFalse(url_is_from_any_domain(url, ['art.co.uk']))
url = 'http://www.Wheele-Bin-Art.co.uk/get/product/123'
self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.CO.UK']))
self.assertTrue(url_is_from_any_domain(url, ['WHEELE-BIN-ART.CO.UK']))
url = 'http://192.169.0.15:8080/mypage.html'
self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080']))
self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15']))

4
scrapy/utils/url.py
View File

@ -17,10 +17,10 @@ from scrapy.utils.python import unicode_to_str
def url_is_from_any_domain(url, domains):
"""Return True if the url belongs to any of the given domains"""
host = parse_url(url).netloc
host = parse_url(url).netloc.lower()
if host:
return any(((host == d) or (host.endswith('.%s' % d)) for d in domains))
return any(((host == d.lower()) or (host.endswith('.%s' % d.lower())) for d in domains))
else:
return False