diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 68f727e48..a39f12afa 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -2,7 +2,7 @@ from scrapy.command import ScrapyCommand from scrapy.http import Request from scrapy.item import BaseItem from scrapy.utils import display -from scrapy.utils.spider import iterate_spider_output +from scrapy.utils.spider import iterate_spider_output, create_spider_for_request from scrapy.utils.url import is_url from scrapy.exceptions import UsageError from scrapy import log @@ -73,7 +73,7 @@ class Command(ScrapyCommand): except KeyError: log.msg('Unable to find spider: %s' % opts.spider, log.ERROR) else: - spider = self.crawler.spiders.create_for_request(request) + spider = create_spider_for_request(self.crawler.spiders, request) if spider: return spider log.msg('Unable to find spider for: %s' % request, log.ERROR) diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index ddf94932b..43d373359 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -33,3 +33,20 @@ class ISpiderQueue(Interface): """Clear the queue. This method can return a deferred. """ + +class ISpiderManager(Interface): + + def from_settings(settings): + """Class method to instantiate from settings""" + + def create(spider_name, **spider_args): + """Returns a new Spider instance for the given spider name, and using + the given spider arguments. If the spider name is not found, it must + raise a KeyError.""" + + def list(): + """Return a list with the names of all spiders available in the + project""" + + def find_by_request(request): + """Returns the list of spiders names that can handle the given request""" diff --git a/scrapy/queue.py b/scrapy/queue.py index fbb486038..d23d43fa4 100644 --- a/scrapy/queue.py +++ b/scrapy/queue.py @@ -2,6 +2,7 @@ from twisted.internet import defer from scrapy.http import Request from scrapy.utils.misc import arg_to_iter +from scrapy.utils.spider import create_spider_for_request from scrapy import log @@ -58,7 +59,7 @@ class ExecutionQueue(object): def append_request(self, request, spider=None, **kwargs): if spider is None: - spider = self._spiders.create_for_request(request, **kwargs) + spider = create_spider_for_request(self._spiders, request, **kwargs) if spider: self.spider_requests.append((spider, [request])) @@ -69,7 +70,8 @@ class ExecutionQueue(object): if url is None: raise ValueError("A url is required") if spider is None: - spider = self._spiders.create_for_request(Request(url), **kwargs) + spider = create_spider_for_request(self._spiders, Request(url), \ + **kwargs) if spider: requests = arg_to_iter(spider.make_requests_from_url(url)) self.spider_requests.append((spider, requests)) diff --git a/scrapy/shell.py b/scrapy/shell.py index f62f19103..cb299948a 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -13,6 +13,7 @@ from scrapy import log from scrapy.item import BaseItem from scrapy.spider import BaseSpider from scrapy.selector import XPathSelector, XmlXPathSelector, HtmlXPathSelector +from scrapy.utils.spider import create_spider_for_request from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser from scrapy.utils.url import any_to_uri @@ -58,7 +59,7 @@ class Shell(object): url = any_to_uri(request_or_url) request = Request(url, dont_filter=True) if spider is None: - spider = self.crawler.spiders.create_for_request(request, \ + spider = create_spider_for_request(self.crawler.spiders, request, \ BaseSpider('default'), log_multiple=True) self.crawler.engine.open_spider(spider) response = None diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py index 8a7af73ac..a398e553b 100644 --- a/scrapy/spidermanager.py +++ b/scrapy/spidermanager.py @@ -3,7 +3,10 @@ SpiderManager is the class which locates and manages all website-specific spiders """ +from zope.interface import implements + from scrapy import log, signals +from scrapy.interfaces import ISpiderManager from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes from scrapy.xlib.pydispatch import dispatcher @@ -11,6 +14,8 @@ from scrapy.xlib.pydispatch import dispatcher class SpiderManager(object): + implements(ISpiderManager) + def __init__(self, spider_modules): self.spider_modules = spider_modules self._spiders = {} @@ -28,41 +33,13 @@ class SpiderManager(object): return cls(settings.getlist('SPIDER_MODULES')) def create(self, spider_name, **spider_kwargs): - """Returns a Spider instance for the given spider name, using the given - spider arguments. If the spider name is not found, it raises a - KeyError. - """ return self._spiders[spider_name](**spider_kwargs) def find_by_request(self, request): - """Returns list of spiders names that match the given Request""" return [name for name, cls in self._spiders.iteritems() - if cls.handles_request(request)] - - def create_for_request(self, request, default_spider=None, \ - log_none=False, log_multiple=False, **spider_kwargs): - """Create a spider to handle the given Request. - - This will look for the spiders that can handle the given request (using - find_by_request) and return a (new) Spider if (and only if) there is - only one Spider able to handle the Request. - - If multiple spiders (or no spider) are found, it will return the - default_spider passed. It can optionally log if multiple or no spiders - are found. - """ - snames = self.find_by_request(request) - if len(snames) == 1: - return self.create(snames[0], **spider_kwargs) - if len(snames) > 1 and log_multiple: - log.msg('More than one spider can handle: %s - %s' % \ - (request, ", ".join(snames)), log.ERROR) - if len(snames) == 0 and log_none: - log.msg('Unable to find spider that handles: %s' % request, log.ERROR) - return default_spider + if cls.handles_request(request)] def list(self): - """Returns list of spiders available.""" return self._spiders.keys() def close_spider(self, spider, reason): diff --git a/scrapy/spiderqueue.py b/scrapy/spiderqueue.py index 780d99e26..03b84fe91 100644 --- a/scrapy/spiderqueue.py +++ b/scrapy/spiderqueue.py @@ -1,5 +1,4 @@ from zope.interface import implements -from zope.interface.verify import verifyObject from scrapy.interfaces import ISpiderQueue from scrapy.utils.sqlite import JsonSqlitePriorityQueue diff --git a/scrapy/tests/test_queue.py b/scrapy/tests/test_queue.py index bf12c91b3..93e166492 100644 --- a/scrapy/tests/test_queue.py +++ b/scrapy/tests/test_queue.py @@ -18,8 +18,8 @@ class TestSpider(BaseSpider): class TestSpiderManager(object): - def create_for_request(self, request, **kwargs): - return TestSpider('create_for_request', **kwargs) + def find_by_request(self, request): + return ['create_for_request'] def create(self, spider_name, **spider_kwargs): return TestSpider(spider_name, **spider_kwargs) diff --git a/scrapy/tests/test_spidermanager/__init__.py b/scrapy/tests/test_spidermanager/__init__.py index 8bee3c4e1..4b46022e6 100644 --- a/scrapy/tests/test_spidermanager/__init__.py +++ b/scrapy/tests/test_spidermanager/__init__.py @@ -3,11 +3,14 @@ import os import weakref import shutil +from zope.interface.verify import verifyObject from twisted.trial import unittest + # ugly hack to avoid cyclic imports of scrapy.spider when running this test # alone import scrapy.spider +from scrapy.interfaces import ISpiderManager from scrapy.spidermanager import SpiderManager from scrapy.http import Request @@ -28,6 +31,9 @@ class SpiderManagerTest(unittest.TestCase): del self.spiderman sys.path.remove(self.tmpdir) + def test_interface(self): + verifyObject(ISpiderManager, self.spiderman) + def test_list(self): self.assertEqual(set(self.spiderman.list()), set(['spider1', 'spider2', 'spider3'])) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 8a30f6dcb..db321dd85 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -1,5 +1,6 @@ import inspect +from scrapy import log from scrapy.item import BaseItem from scrapy.utils.misc import arg_to_iter @@ -21,3 +22,26 @@ def iter_spider_classes(module): obj.__module__ == module.__name__ and \ getattr(obj, 'name', None): yield obj + +def create_spider_for_request(spidermanager, request, default_spider=None, \ + log_none=False, log_multiple=False, **spider_kwargs): + """Create a spider to handle the given Request. + + This will look for the spiders that can handle the given request (using + the spider manager) and return a (new) Spider if (and only if) there is + only one Spider able to handle the Request. + + If multiple spiders (or no spider) are found, it will return the + default_spider passed. It can optionally log if multiple or no spiders + are found. + """ + snames = spidermanager.find_by_request(request) + if len(snames) == 1: + return spidermanager.create(snames[0], **spider_kwargs) + if len(snames) > 1 and log_multiple: + log.msg('More than one spider can handle: %s - %s' % \ + (request, ", ".join(snames)), log.ERROR) + if len(snames) == 0 and log_none: + log.msg('Unable to find spider that handles: %s' % request, log.ERROR) + return default_spider +