RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add tests for url_is_from_spider() with allowed_domains

This commit is contained in:
Steven Almeroth 2013-03-10 16:41:27 -06:00
parent b48ec1dce4
commit a613e15154
1 changed files with 20 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
scrapy/tests/test_utils_url.py
View File

@ -48,6 +48,16 @@ class UrlUtilsTest(unittest.TestCase):
self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider))
self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider))
spider = BaseSpider(name='example.com', allowed_domains=set(('example.com', 'example.net')))
self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
spider = BaseSpider(name='example.com', allowed_domains=('example.com', 'example.net'))
self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', spider))
spider = BaseSpider(name='example.com', allowed_domains={'example.org': None})
self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', spider))
def test_url_is_from_spider_with_allowed_domains_class_attributes(self):
class MySpider(BaseSpider):
name = 'example.com'
@ -59,6 +69,16 @@ class UrlUtilsTest(unittest.TestCase):
self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
class MySetSpider(BaseSpider):
name = 'example.com'
allowed_domains = set(('example.org', 'example.net'))
self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySetSpider))
self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySetSpider))
self.assertTrue(url_is_from_spider('http://example.com/some/page.html', MySetSpider))
self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', MySetSpider))
self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySetSpider))
self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySetSpider))
def test_canonicalize_url(self):
# simplest case
self.assertEqual(canonicalize_url("http://www.example.com/"),