diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 5d7d5ceae..63b8054bc 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -58,11 +58,11 @@ class Command(ScrapyCommand): for url in urls: spider_names = spiders.find_by_request(Request(url)) if not spider_names: - log.msg('Could not find spider for url: %s' % url, + log.msg('Could not find spider that handles url: %s' % url, log.ERROR) elif len(spider_names) > 1: - log.msg('More than one spider found for url: %s' % url, - log.ERROR) + log.msg('More than one spider can handle url: %s - %s' % \ + (url, ", ".join(spider_names)), log.ERROR) else: spider_urls[spider_names[0]].append(url) return spider_urls.items() diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index 7b54c8173..111b97bfe 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -7,7 +7,6 @@ import inspect from scrapy import log from scrapy.conf import settings -from scrapy.utils.url import url_is_from_spider from scrapy.utils.misc import walk_modules from scrapy.spider import BaseSpider @@ -28,7 +27,7 @@ class SpiderManager(object): def find_by_request(self, request): """Returns list of spiders names that match the given Request""" return [name for name, spider in self._spiders.iteritems() - if url_is_from_spider(request.url, spider)] + if spider.handles_request(request)] def create_for_request(self, request, default_spider=None, \ log_none=False, log_multiple=False, **spider_kwargs): @@ -46,9 +45,10 @@ class SpiderManager(object): if len(snames) == 1: return self.create(snames[0], **spider_kwargs) if len(snames) > 1 and log_multiple: - log.msg('More than one spider found for: %s' % request, log.ERROR) + log.msg('More than one spider can handle: %s - %s' % \ + (request, ", ".join(snames)), log.ERROR) if len(snames) == 0 and log_none: - log.msg('Unable to find spider for: %s' % request, log.ERROR) + log.msg('Unable to find spider that handles: %s' % request, log.ERROR) return default_spider def list(self): diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index a58f0a0c9..4ca4305d0 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -10,6 +10,7 @@ from scrapy import log from scrapy.http import Request from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref +from scrapy.utils.url import url_is_from_spider class BaseSpider(object_ref): @@ -68,6 +69,10 @@ class BaseSpider(object_ref): def parse(self, response): raise NotImplementedError + @classmethod + def handles_request(cls, request): + return url_is_from_spider(request.url, cls) + def __str__(self): return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self)) diff --git a/scrapy/tests/test_contrib_spidermanager/__init__.py b/scrapy/tests/test_contrib_spidermanager/__init__.py index 930ceed54..c7a05d560 100644 --- a/scrapy/tests/test_contrib_spidermanager/__init__.py +++ b/scrapy/tests/test_contrib_spidermanager/__init__.py @@ -33,7 +33,7 @@ class SpiderManagerTest(unittest.TestCase): def test_list(self): self.assertEqual(set(self.spiderman.list()), - set(['spider1', 'spider2'])) + set(['spider1', 'spider2', 'spider3'])) def test_create(self): spider1 = self.spiderman.create("spider1") @@ -51,6 +51,10 @@ class SpiderManagerTest(unittest.TestCase): set(['spider1', 'spider2'])) self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')), []) + self.assertEqual(self.spiderman.find_by_request(Request('http://spider3.com')), + []) + self.assertEqual(self.spiderman.find_by_request(Request('http://spider3.com/onlythis')), + ['spider3']) def test_load_spider_module(self): self.spiderman.load(['scrapy.tests.test_contrib_spidermanager.test_spiders.spider1']) diff --git a/scrapy/tests/test_contrib_spidermanager/test_spiders/spider3.py b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider3.py new file mode 100644 index 000000000..ac9229c8c --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider3.py @@ -0,0 +1,9 @@ +from scrapy.spider import BaseSpider + +class Spider3(BaseSpider): + name = "spider3" + allowed_domains = ['spider3.com'] + + @classmethod + def handles_request(cls, request): + return request.url == 'http://spider3.com/onlythis'