From b1f011d7408343aabdf89c2c25fcde9b3c2b38a2 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 24 Feb 2012 02:09:02 -0200 Subject: [PATCH] use netloc instead of hostname in url_is_from_any_domain(). closes #50 --- scrapy/tests/test_utils_url.py | 4 ++++ scrapy/utils/url.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index fc17ccf6a..ec27bddea 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -15,6 +15,10 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk'])) self.assertFalse(url_is_from_any_domain(url, ['art.co.uk'])) + url = 'http://192.169.0.15:8080/mypage.html' + self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080'])) + self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15'])) + url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29' self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com'])) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 8a309bdf6..cae73f398 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -15,7 +15,7 @@ from scrapy.utils.python import unicode_to_str def url_is_from_any_domain(url, domains): """Return True if the url belongs to any of the given domains""" - host = parse_url(url).hostname + host = parse_url(url).netloc if host: return any(((host == d) or (host.endswith('.%s' % d)) for d in domains))