Added handles_request() class method to BaseSpider - closes #191

scrapy/commands/crawl.py

@@ -58,11 +58,11 @@ class Command(ScrapyCommand):
         for url in urls:
             spider_names = spiders.find_by_request(Request(url))
             if not spider_names:
-                log.msg('Could not find spider for url: %s' % url,
+                log.msg('Could not find spider that handles url: %s' % url,
                         log.ERROR)
             elif len(spider_names) > 1:
-                log.msg('More than one spider found for url: %s' % url,
-                        log.ERROR)
+                log.msg('More than one spider can handle url: %s - %s' % \
+                    (url, ", ".join(spider_names)), log.ERROR)
             else:
                 spider_urls[spider_names[0]].append(url)
         return spider_urls.items()

scrapy/contrib/spidermanager.py

@@ -7,7 +7,6 @@ import inspect
 
 from scrapy import log
 from scrapy.conf import settings
-from scrapy.utils.url import url_is_from_spider
 from scrapy.utils.misc import walk_modules
 from scrapy.spider import BaseSpider
 
@@ -28,7 +27,7 @@ class SpiderManager(object):
     def find_by_request(self, request):
         """Returns list of spiders names that match the given Request"""
         return [name for name, spider in self._spiders.iteritems()
-                if url_is_from_spider(request.url, spider)]
+                if spider.handles_request(request)]
 
     def create_for_request(self, request, default_spider=None, \
             log_none=False, log_multiple=False, **spider_kwargs):
@@ -46,9 +45,10 @@ class SpiderManager(object):
         if len(snames) == 1:
             return self.create(snames[0], **spider_kwargs)
         if len(snames) > 1 and log_multiple:
-            log.msg('More than one spider found for: %s' % request, log.ERROR)
+            log.msg('More than one spider can handle: %s - %s' % \
+                (request, ", ".join(snames)), log.ERROR)
         if len(snames) == 0 and log_none:
-            log.msg('Unable to find spider for: %s' % request, log.ERROR)
+            log.msg('Unable to find spider that handles: %s' % request, log.ERROR)
         return default_spider
 
     def list(self):

scrapy/spider/models.py

@@ -10,6 +10,7 @@ from scrapy import log
 from scrapy.http import Request
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.trackref import object_ref
+from scrapy.utils.url import url_is_from_spider
 
 
 class BaseSpider(object_ref):
@@ -68,6 +69,10 @@ class BaseSpider(object_ref):
     def parse(self, response):
         raise NotImplementedError
 
+    @classmethod
+    def handles_request(cls, request):
+        return url_is_from_spider(request.url, cls)
+
     def __str__(self):
         return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self))
 

scrapy/tests/test_contrib_spidermanager/__init__.py

@@ -33,7 +33,7 @@ class SpiderManagerTest(unittest.TestCase):
 
     def test_list(self):
         self.assertEqual(set(self.spiderman.list()),
-            set(['spider1', 'spider2']))
+            set(['spider1', 'spider2', 'spider3']))
 
     def test_create(self):
         spider1 = self.spiderman.create("spider1")
@@ -51,6 +51,10 @@ class SpiderManagerTest(unittest.TestCase):
             set(['spider1', 'spider2']))
         self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')),
             [])
+        self.assertEqual(self.spiderman.find_by_request(Request('http://spider3.com')),
+            [])
+        self.assertEqual(self.spiderman.find_by_request(Request('http://spider3.com/onlythis')),
+            ['spider3'])
 
     def test_load_spider_module(self):
         self.spiderman.load(['scrapy.tests.test_contrib_spidermanager.test_spiders.spider1'])

scrapy/tests/test_contrib_spidermanager/test_spiders/spider3.py

@@ -0,0 +1,9 @@
+from scrapy.spider import BaseSpider
+
+class Spider3(BaseSpider):
+    name = "spider3"
+    allowed_domains = ['spider3.com']
+
+    @classmethod
+    def handles_request(cls, request):
+        return request.url == 'http://spider3.com/onlythis'