From b563e56363aabe1d7fb6195151b0f5443f53e964 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 16 Aug 2010 12:25:51 -0300 Subject: [PATCH] fixed bug in url_is_from_spider() when no allowed_domains class attribute is present --- scrapy/tests/test_utils_url.py | 28 +++++++++++++++++++++++++++- scrapy/utils/url.py | 3 ++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index 058b94276..8acb6fa61 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -19,7 +19,22 @@ class UrlUtilsTest(unittest.TestCase): self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com'])) - def test_url_is_from_any_domain(self): + def test_url_is_from_spider(self): + spider = BaseSpider(name='example.com') + self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) + self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider)) + self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', spider)) + self.assertFalse(url_is_from_spider('http://www.example.net/some/page.html', spider)) + + def test_url_is_from_spider_class_attributes(self): + class MySpider(BaseSpider): + name = 'example.com' + self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider)) + self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider)) + self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', MySpider)) + self.assertFalse(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) + + def test_url_is_from_spider_with_allowed_domains(self): spider = BaseSpider(name='example.com', allowed_domains=['example.org', 'example.net']) self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider)) @@ -28,6 +43,17 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider)) + def test_url_is_from_spider_with_allowed_domains_class_attributes(self): + class MySpider(BaseSpider): + name = 'example.com' + allowed_domains = ['example.org', 'example.net'] + self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider)) + self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider)) + self.assertTrue(url_is_from_spider('http://example.com/some/page.html', MySpider)) + self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', MySpider)) + self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) + self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider)) + def test_urljoin_rfc(self): self.assertEqual(urljoin_rfc('http://example.com/some/path', 'newpath/test'), 'http://example.com/some/newpath/test') diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 31e21789e..e82638266 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -22,7 +22,8 @@ def url_is_from_any_domain(url, domains): def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - return url_is_from_any_domain(url, [spider.name] + spider.allowed_domains) + return url_is_from_any_domain(url, [spider.name] + \ + getattr(spider, 'allowed_domains', [])) def urljoin_rfc(base, ref, encoding='utf-8'): """Same as urlparse.urljoin but supports unicode values in base and ref