From 9fccc113631fee4a3148fe688164d7b750ace45f Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 22 Aug 2010 02:15:11 -0300 Subject: [PATCH] Moved scrapy.extension.extensions singleton to a "extensions" attribute of the scrapy.project.crawler singleton. Refs #189 --- docs/topics/extensions.rst | 8 ++++---- docs/topics/telnetconsole.rst | 8 ++++---- scrapy/contrib/webservice/extensions.py | 6 ++++-- scrapy/crawler.py | 10 +++++----- scrapy/extension.py | 2 -- scrapy/project.py | 8 +++++--- scrapy/telnet.py | 3 +-- 7 files changed, 23 insertions(+), 22 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 13b36958d..539d4ef1f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -71,7 +71,7 @@ Even though it's not usually needed, you can access extension objects through the :ref:`topics-extensions-ref-manager` which is populated when extensions are loaded. For example, to access the ``WebService`` extension:: - from scrapy.extension import extensions + from scrapy.project import extensions webservice_extension = extensions.enabled['WebService'] .. see also:: @@ -128,7 +128,7 @@ how you :ref:`configure the downloader middlewares The Extension Manager is a singleton object, which is instantiated at module loading time and can be accessed like this:: - from scrapy.extension import extensions + from scrapy.project import extensions .. attribute:: loaded @@ -140,7 +140,7 @@ how you :ref:`configure the downloader middlewares A dict with the enabled extensions. The keys are the extension class names, and the values are the extension objects. Example:: - >>> from scrapy.extension import extensions + >>> from scrapy.project import extensions >>> extensions.load() >>> print extensions.enabled {'CoreStats': , @@ -153,7 +153,7 @@ how you :ref:`configure the downloader middlewares and the values are the extension class paths (because objects are never instantiated for disabled extensions). Example:: - >>> from scrapy.extension import extensions + >>> from scrapy.project import extensions >>> extensions.load() >>> print extensions.disabled {'MemoryDebugger': 'scrapy.contrib.memdebug.MemoryDebugger', diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 770b2ba53..831bcd6d4 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -43,13 +43,13 @@ convenience: +----------------+-------------------------------------------------------------------+ | Shortcut | Description | +================+===================================================================+ -| ``engine`` | the Scrapy engine object (``scrapy.core.engine.scrapyengine``) | -+----------------+-------------------------------------------------------------------+ | ``manager`` | the Project Crawler object (``scrapy.project.crawler``) | +----------------+-------------------------------------------------------------------+ -| ``extensions`` | the extensions object (``scrapy.extension.extensions``) | +| ``engine`` | the Scrapy engine object (``scrapy.project.crawler.engine``) | +----------------+-------------------------------------------------------------------+ -| ``stats`` | the Scrapy stats object (``scrapy.stats.stats``) | +| ``extensions`` | the Extension Manager (``scrapy.project.crawler.extensions``) | ++----------------+-------------------------------------------------------------------+ +| ``stats`` | the Stats Collector (``scrapy.stats.stats``) | +----------------+-------------------------------------------------------------------+ | ``settings`` | the Scrapy settings object (``scrapy.conf.settings``) | +----------------+-------------------------------------------------------------------+ diff --git a/scrapy/contrib/webservice/extensions.py b/scrapy/contrib/webservice/extensions.py index e18186cd1..352fca7ac 100644 --- a/scrapy/contrib/webservice/extensions.py +++ b/scrapy/contrib/webservice/extensions.py @@ -1,10 +1,12 @@ from scrapy.webservice import JsonRpcResource -from scrapy.extension import extensions +from scrapy.project import crawler class ExtensionsResource(JsonRpcResource): ws_name = 'extensions' - def __init__(self, _extensions=extensions): + def __init__(self, _extensions=None): + if _extensions is None: + _extensions = crawler.extensions JsonRpcResource.__init__(self) self._target = _extensions diff --git a/scrapy/crawler.py b/scrapy/crawler.py index a743a86a5..719799aea 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,16 +4,16 @@ from twisted.internet import reactor, defer from scrapy.core.engine import ExecutionEngine from scrapy.core.queue import ExecutionQueue -from scrapy.extension import extensions from scrapy import log from scrapy.utils.ossignal import install_shutdown_handlers, signal_names class Crawler(object): - def __init__(self, spiders): + def __init__(self, spiders, extensions): self.configured = False self.control_reactor = True + self.extensions = extensions self.spiders = spiders self.engine = ExecutionEngine(self) @@ -24,11 +24,11 @@ class Crawler(object): if not log.started: log.start() - if not extensions.loaded: - extensions.load() + if not self.extensions.loaded: + self.extensions.load() if not self.spiders.loaded: self.spiders.load() - log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys()), + log.msg("Enabled extensions: %s" % ", ".join(self.extensions.enabled.iterkeys()), level=log.DEBUG) self.queue = queue or ExecutionQueue() diff --git a/scrapy/extension.py b/scrapy/extension.py index ede22447f..771e5ecfa 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -38,5 +38,3 @@ class ExtensionManager(object): def reload(self): self.load() - -extensions = ExtensionManager() diff --git a/scrapy/project.py b/scrapy/project.py index 9c01e0e3e..ca59f7dac 100644 --- a/scrapy/project.py +++ b/scrapy/project.py @@ -1,6 +1,8 @@ -from scrapy.conf import settings -from scrapy.utils.misc import load_object from scrapy.crawler import Crawler +from scrapy.extension import ExtensionManager +from scrapy.utils.misc import load_object +from scrapy.conf import settings _spiders = load_object(settings['SPIDER_MANAGER_CLASS'])() -crawler = Crawler(_spiders) +_extensions = ExtensionManager() +crawler = Crawler(_spiders, _extensions) diff --git a/scrapy/telnet.py b/scrapy/telnet.py index c5dbced59..f4ae9673d 100644 --- a/scrapy/telnet.py +++ b/scrapy/telnet.py @@ -10,7 +10,6 @@ from twisted.conch import manhole, telnet from twisted.conch.insults import insults from twisted.internet import reactor, protocol -from scrapy.extension import extensions from scrapy.exceptions import NotConfigured from scrapy.project import crawler from scrapy.stats import stats @@ -49,7 +48,7 @@ class TelnetConsole(protocol.ServerFactory): telnet_vars = { 'engine': crawler.engine, 'manager': crawler, - 'extensions': extensions, + 'extensions': crawler.extensions, 'stats': stats, 'spiders': crawler.spiders, 'settings': settings,