From b2878fbbef321dc7cfc5fb0f71714e2c8fd880ce Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 2 Aug 2010 12:16:36 -0300 Subject: [PATCH] scrapy.utils.url_is_from_spider() - Consider spider name as possible domain for matching in scrapy.utils.url_is_from_spider() --- scrapy/tests/test_utils_url.py | 13 +++++++++++-- scrapy/utils/url.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index af61b8e7f..058b94276 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -1,8 +1,8 @@ import unittest +from scrapy.spider import BaseSpider from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, \ url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url, \ - urljoin_rfc - + urljoin_rfc, url_is_from_spider class UrlUtilsTest(unittest.TestCase): @@ -19,6 +19,15 @@ class UrlUtilsTest(unittest.TestCase): self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com'])) + def test_url_is_from_any_domain(self): + spider = BaseSpider(name='example.com', allowed_domains=['example.org', 'example.net']) + self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) + self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider)) + self.assertTrue(url_is_from_spider('http://example.com/some/page.html', spider)) + self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', spider)) + self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider)) + self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider)) + def test_urljoin_rfc(self): self.assertEqual(urljoin_rfc('http://example.com/some/path', 'newpath/test'), 'http://example.com/some/newpath/test') diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 196c03727..31e21789e 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -22,7 +22,7 @@ def url_is_from_any_domain(url, domains): def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - return url_is_from_any_domain(url, spider.allowed_domains) + return url_is_from_any_domain(url, [spider.name] + spider.allowed_domains) def urljoin_rfc(base, ref, encoding='utf-8'): """Same as urlparse.urljoin but supports unicode values in base and ref