RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use netloc instead of hostname in url_is_from_any_domain(). closes #50

This commit is contained in:
Pablo Hoffman 2012-02-24 02:09:02 -02:00
parent 08d2c2b9ee
commit b1f011d740
2 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/tests/test_utils_url.py
View File

@ -15,6 +15,10 @@ class UrlUtilsTest(unittest.TestCase):
self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk']))
self.assertFalse(url_is_from_any_domain(url, ['art.co.uk']))
url = 'http://192.169.0.15:8080/mypage.html'
self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080']))
self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15']))
url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29'
self.assertFalse(url_is_from_any_domain(url, ['testdomain.com']))
self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com']))

2
scrapy/utils/url.py
View File

@ -15,7 +15,7 @@ from scrapy.utils.python import unicode_to_str
def url_is_from_any_domain(url, domains):
"""Return True if the url belongs to any of the given domains"""
host = parse_url(url).hostname
host = parse_url(url).netloc
if host:
return any(((host == d) or (host.endswith('.%s' % d)) for d in domains))