RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added test for new url_has_any_extension function

This commit is contained in:
Yann Defretin 2022-03-16 22:32:05 +01:00
parent 78ba4b033b
commit 5b4b8b6fb1
No known key found for this signature in database
GPG Key ID: B426EC9CACC79704
1 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tests/test_utils_url.py
View File

@ -1,6 +1,8 @@
import unittest
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.spiders import Spider
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.url import (
add_http_if_no_scheme,
guess_scheme,
@ -8,9 +10,9 @@ from scrapy.utils.url import (
strip_url,
url_is_from_any_domain,
url_is_from_spider,
url_has_any_extension,
)
__doctests__ = ['scrapy.utils.url']
@ -81,6 +83,15 @@ class UrlUtilsTest(unittest.TestCase):
self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
def test_url_has_any_extension(self):
deny_extensions = {'.' + e for e in arg_to_iter(IGNORED_EXTENSIONS)}
self.assertTrue(url_has_any_extension("http://www.example.com/archive.tar.gz", deny_extensions))
self.assertTrue(url_has_any_extension("http://www.example.com/page.doc", deny_extensions))
self.assertTrue(url_has_any_extension("http://www.example.com/page.pdf", deny_extensions))
self.assertFalse(url_has_any_extension("http://www.example.com/page.htm", deny_extensions))
self.assertFalse(url_has_any_extension("http://www.example.com/", deny_extensions))
self.assertFalse(url_has_any_extension("http://www.example.com/page.doc.html", deny_extensions))
class AddHttpIfNoScheme(unittest.TestCase):