From 27ca25472fd3fb5b540d83ea95a9eada5ae13c61 Mon Sep 17 00:00:00 2001 From: Zuhao Wan Date: Mon, 11 Feb 2013 17:19:31 +0800 Subject: [PATCH] Fix url_has_any_extension bug --- scrapy/tests/test_utils_url.py | 10 +++++++++- scrapy/utils/url.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index f38145674..68cf424ed 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -1,6 +1,7 @@ import unittest from scrapy.spider import BaseSpider -from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url +from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url, url_has_any_extension +from scrapy.linkextractor import IGNORED_EXTENSIONS __doctests__ = ['scrapy.utils.url'] @@ -157,6 +158,13 @@ class UrlUtilsTest(unittest.TestCase): self.assertEqual(canonicalize_url("http://foo.com/AC%2FDC/"), "http://foo.com/AC%2FDC/") + def test_url_has_any_extension(self): + self.assertTrue(url_has_any_extension("http://www.example.com/page.doc", IGNORED_EXTENSIONS)) + self.assertTrue(url_has_any_extension("http://www.example.com/page.pdf", IGNORED_EXTENSIONS)) + self.assertFalse(url_has_any_extension("http://www.example.com/page.htm", IGNORED_EXTENSIONS)) + self.assertFalse(url_has_any_extension("http://www.example.com/", IGNORED_EXTENSIONS)) + self.assertFalse(url_has_any_extension("http://www.example.com/page.doc.html", IGNORED_EXTENSIONS)) + if __name__ == "__main__": unittest.main() diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index bc8236e1a..16868b027 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -28,7 +28,7 @@ def url_is_from_spider(url, spider): getattr(spider, 'allowed_domains', [])) def url_has_any_extension(url, extensions): - return posixpath.splitext(parse_url(url).path)[1].lower() in extensions + return posixpath.splitext(parse_url(url).path)[1].replace('.', '').lower() in extensions def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \ encoding=None):