diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index a6a2a9e8b..4d5e9ae82 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -5,7 +5,6 @@ library. Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ -import posixpath import re from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse @@ -31,8 +30,9 @@ def url_is_from_spider(url, spider): def url_has_any_extension(url, extensions): - return posixpath.splitext(parse_url(url).path)[1].lower() in extensions - + """Return True if the url ends with one of the extensions provided""" + lowercase_path = parse_url(url).path.lower() + return any(lowercase_path.endswith(ext) for ext in extensions) def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 144c7bd76..58e2be622 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,6 +1,8 @@ import unittest +from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider +from scrapy.utils.misc import arg_to_iter from scrapy.utils.url import ( add_http_if_no_scheme, guess_scheme, @@ -8,9 +10,9 @@ from scrapy.utils.url import ( strip_url, url_is_from_any_domain, url_is_from_spider, + url_has_any_extension, ) - __doctests__ = ['scrapy.utils.url'] @@ -81,6 +83,15 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider)) + def test_url_has_any_extension(self): + deny_extensions = {'.' + e for e in arg_to_iter(IGNORED_EXTENSIONS)} + self.assertTrue(url_has_any_extension("http://www.example.com/archive.tar.gz", deny_extensions)) + self.assertTrue(url_has_any_extension("http://www.example.com/page.doc", deny_extensions)) + self.assertTrue(url_has_any_extension("http://www.example.com/page.pdf", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/page.htm", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/page.doc.html", deny_extensions)) + class AddHttpIfNoScheme(unittest.TestCase):