scrapy.utils.url_is_from_spider() - Consider spider name as possible domain for matching in scrapy.utils.url_is_from_spider()

scrapy/tests/test_utils_url.py

@@ -1,8 +1,8 @@
 import unittest
+from scrapy.spider import BaseSpider
 from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, \
     url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url, \
-    urljoin_rfc
-
+    urljoin_rfc, url_is_from_spider
 
 class UrlUtilsTest(unittest.TestCase):
 
@@ -19,6 +19,15 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertFalse(url_is_from_any_domain(url, ['testdomain.com']))
         self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com']))
 
+    def test_url_is_from_any_domain(self):
+        spider = BaseSpider(name='example.com', allowed_domains=['example.org', 'example.net'])
+        self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
+        self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider))
+        self.assertTrue(url_is_from_spider('http://example.com/some/page.html', spider))
+        self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', spider))
+        self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider))
+        self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider))
+
     def test_urljoin_rfc(self):
         self.assertEqual(urljoin_rfc('http://example.com/some/path', 'newpath/test'),
                                      'http://example.com/some/newpath/test')

scrapy/utils/url.py

@@ -22,7 +22,7 @@ def url_is_from_any_domain(url, domains):
 
 def url_is_from_spider(url, spider):
     """Return True if the url belongs to the given spider"""
-    return url_is_from_any_domain(url, spider.allowed_domains)
+    return url_is_from_any_domain(url, [spider.name] + spider.allowed_domains)
 
 def urljoin_rfc(base, ref, encoding='utf-8'):
     """Same as urlparse.urljoin but supports unicode values in base and ref