diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 719799aea..d89cf601f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,16 +4,17 @@ from twisted.internet import reactor, defer from scrapy.core.engine import ExecutionEngine from scrapy.core.queue import ExecutionQueue +from scrapy.extension import ExtensionManager from scrapy import log from scrapy.utils.ossignal import install_shutdown_handlers, signal_names class Crawler(object): - def __init__(self, spiders, extensions): + def __init__(self, settings, spiders): self.configured = False self.control_reactor = True - self.extensions = extensions + self.settings = settings self.spiders = spiders self.engine = ExecutionEngine(self) @@ -24,12 +25,9 @@ class Crawler(object): if not log.started: log.start() - if not self.extensions.loaded: - self.extensions.load() + self.extensions = ExtensionManager.from_settings(self.settings) if not self.spiders.loaded: self.spiders.load() - log.msg("Enabled extensions: %s" % ", ".join(self.extensions.enabled.iterkeys()), - level=log.DEBUG) self.queue = queue or ExecutionQueue() self.engine.configure(self._spider_closed) diff --git a/scrapy/extension.py b/scrapy/extension.py index 771e5ecfa..f68b1ba68 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -1,40 +1,16 @@ """ -This module contains the ExtensionManager which takes care of loading and -keeping track of all enabled extensions. It also contains an instantiated -ExtensionManager (extensions) to be used as singleton. +The Extension Manager + +See documentation in docs/topics/extensions.rst """ -from scrapy.exceptions import NotConfigured -from scrapy.utils.misc import load_object +from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list -from scrapy import log -from scrapy.conf import settings -class ExtensionManager(object): +class ExtensionManager(MiddlewareManager): - def __init__(self): - self.loaded = False - self.enabled = {} - self.disabled = {} + component_name = 'extension' - def load(self): - """ - Load enabled extensions in settings module - """ - - self.loaded = False - self.enabled.clear() - self.disabled.clear() - extlist = build_component_list(settings['EXTENSIONS_BASE'], \ + @classmethod + def _get_mwlist_from_settings(cls, settings): + return build_component_list(settings['EXTENSIONS_BASE'], \ settings['EXTENSIONS']) - for extension_path in extlist: - try: - cls = load_object(extension_path) - self.enabled[cls.__name__] = cls() - except NotConfigured, e: - self.disabled[cls.__name__] = extension_path - if e.args: - log.msg(e) - self.loaded = True - - def reload(self): - self.load() diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 1a5d72c21..818dcdb87 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -35,7 +35,7 @@ class MiddlewareManager(object): except NotConfigured, e: if e.args: log.msg(e) - enabled = [type(x).__name__ for x in middlewares] + enabled = [x.__class__.__name__ for x in middlewares] log.msg("Enabled %ss: %s" % (cls.component_name, ", ".join(enabled)), \ level=log.DEBUG) return cls(*middlewares) diff --git a/scrapy/project.py b/scrapy/project.py index ca59f7dac..038ad66e2 100644 --- a/scrapy/project.py +++ b/scrapy/project.py @@ -1,8 +1,6 @@ from scrapy.crawler import Crawler -from scrapy.extension import ExtensionManager from scrapy.utils.misc import load_object from scrapy.conf import settings _spiders = load_object(settings['SPIDER_MANAGER_CLASS'])() -_extensions = ExtensionManager() -crawler = Crawler(_spiders, _extensions) +crawler = Crawler(settings, _spiders)