From 78ba4b033b016be7fbb22bfa9e6d5d389380e6d4 Mon Sep 17 00:00:00 2001 From: Yann Defretin Date: Wed, 16 Mar 2022 15:14:24 +0100 Subject: [PATCH 1/3] fixed detection of extension like ".tar.gz" in URL --- scrapy/utils/url.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index a6a2a9e8b..bae5a9433 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -5,7 +5,6 @@ library. Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ -import posixpath import re from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse @@ -31,8 +30,8 @@ def url_is_from_spider(url, spider): def url_has_any_extension(url, extensions): - return posixpath.splitext(parse_url(url).path)[1].lower() in extensions - + """Return True if the url ends with one of the extensions provided""" + return any(parse_url(url).path.lower().endswith(ext) for ext in extensions) def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already From 5b4b8b6fb12874d4a0a11c261341639c9af95b10 Mon Sep 17 00:00:00 2001 From: Yann Defretin Date: Wed, 16 Mar 2022 22:32:05 +0100 Subject: [PATCH 2/3] added test for new url_has_any_extension function --- tests/test_utils_url.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 144c7bd76..58e2be622 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,6 +1,8 @@ import unittest +from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider +from scrapy.utils.misc import arg_to_iter from scrapy.utils.url import ( add_http_if_no_scheme, guess_scheme, @@ -8,9 +10,9 @@ from scrapy.utils.url import ( strip_url, url_is_from_any_domain, url_is_from_spider, + url_has_any_extension, ) - __doctests__ = ['scrapy.utils.url'] @@ -81,6 +83,15 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider)) + def test_url_has_any_extension(self): + deny_extensions = {'.' + e for e in arg_to_iter(IGNORED_EXTENSIONS)} + self.assertTrue(url_has_any_extension("http://www.example.com/archive.tar.gz", deny_extensions)) + self.assertTrue(url_has_any_extension("http://www.example.com/page.doc", deny_extensions)) + self.assertTrue(url_has_any_extension("http://www.example.com/page.pdf", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/page.htm", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/page.doc.html", deny_extensions)) + class AddHttpIfNoScheme(unittest.TestCase): From 0905d42e33871e976760d880316210d4953cd5df Mon Sep 17 00:00:00 2001 From: Yann Defretin Date: Thu, 17 Mar 2022 11:19:09 +0100 Subject: [PATCH 3/3] refactored url_has_any_extension function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/url.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index bae5a9433..4d5e9ae82 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -31,7 +31,8 @@ def url_is_from_spider(url, spider): def url_has_any_extension(url, extensions): """Return True if the url ends with one of the extensions provided""" - return any(parse_url(url).path.lower().endswith(ext) for ext in extensions) + lowercase_path = parse_url(url).path.lower() + return any(lowercase_path.endswith(ext) for ext in extensions) def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already