From 755235b5686890d410d7dbda1a0ba13c5e275ea9 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Fri, 13 Feb 2009 17:21:50 +0000 Subject: [PATCH] utils: renamed load_class function as load_object --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40853 --- scrapy/trunk/scrapy/command/commands/shell.py | 4 +-- .../scrapy/contrib/history/middleware.py | 4 +-- .../spidermiddleware/duplicatesfilter.py | 4 +-- .../contrib_exp/cluster/worker/manager.py | 4 +-- .../scrapy/core/downloader/middleware.py | 4 +-- .../core/downloader/responsetypes/__init__.py | 4 +-- scrapy/trunk/scrapy/core/manager.py | 6 ++-- scrapy/trunk/scrapy/extension/__init__.py | 4 +-- scrapy/trunk/scrapy/item/pipeline.py | 4 +-- scrapy/trunk/scrapy/spider/manager.py | 4 +-- scrapy/trunk/scrapy/spider/middleware.py | 4 +-- scrapy/trunk/scrapy/utils/misc.py | 29 ++++++++++++------- 12 files changed, 41 insertions(+), 34 deletions(-) diff --git a/scrapy/trunk/scrapy/command/commands/shell.py b/scrapy/trunk/scrapy/command/commands/shell.py index 541f8be8a..9bc5d2a93 100644 --- a/scrapy/trunk/scrapy/command/commands/shell.py +++ b/scrapy/trunk/scrapy/command/commands/shell.py @@ -5,7 +5,7 @@ import scrapy from scrapy.command import ScrapyCommand from scrapy.spider import spiders from scrapy.xpath import XmlXPathSelector, HtmlXPathSelector -from scrapy.utils.misc import load_class +from scrapy.utils.misc import load_object from scrapy.extension import extensions from scrapy.conf import settings from scrapy.core.manager import scrapymanager @@ -85,7 +85,7 @@ class Command(ScrapyCommand): return True def generate_vars(self, url, response): - itemcls = load_class(settings['DEFAULT_ITEM_CLASS']) + itemcls = load_object(settings['DEFAULT_ITEM_CLASS']) item = itemcls() self.vars['item'] = item if url: diff --git a/scrapy/trunk/scrapy/contrib/history/middleware.py b/scrapy/trunk/scrapy/contrib/history/middleware.py index 710743128..21b6d5aa6 100644 --- a/scrapy/trunk/scrapy/contrib/history/middleware.py +++ b/scrapy/trunk/scrapy/contrib/history/middleware.py @@ -3,7 +3,7 @@ from datetime import datetime from pydispatch import dispatcher -from scrapy.utils.misc import load_class +from scrapy.utils.misc import load_object from scrapy.core import signals from scrapy import log from scrapy.core.exceptions import NotConfigured, IgnoreRequest @@ -16,7 +16,7 @@ class HistoryMiddleware(object): MIN_PROCESS_UNCHANGED_DAYS = 12 def __init__(self): - historycls = load_class(settings['MEMORYSTORE']) + historycls = load_object(settings['MEMORYSTORE']) if not historycls: raise NotConfigured self.historydata = historycls() diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py index ec48222dd..d5f03b429 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py @@ -8,7 +8,7 @@ from scrapy.core import signals from scrapy.http import Request from scrapy.core.exceptions import NotConfigured from scrapy.utils.request import request_fingerprint -from scrapy.utils.misc import load_class +from scrapy.utils.misc import load_object from scrapy.conf import settings from scrapy import log @@ -34,7 +34,7 @@ class DuplicatesFilterMiddleware(object): if not clspath: raise NotConfigured - self.filter = load_class(clspath)() + self.filter = load_object(clspath)() dispatcher.connect(self.filter.open, signals.domain_open) dispatcher.connect(self.filter.close, signals.domain_closed) diff --git a/scrapy/trunk/scrapy/contrib_exp/cluster/worker/manager.py b/scrapy/trunk/scrapy/contrib_exp/cluster/worker/manager.py index 901721e32..7f448f626 100644 --- a/scrapy/trunk/scrapy/contrib_exp/cluster/worker/manager.py +++ b/scrapy/trunk/scrapy/contrib_exp/cluster/worker/manager.py @@ -11,7 +11,7 @@ from twisted.spread import pb from scrapy import log from scrapy.core.engine import scrapyengine from scrapy.core.exceptions import NotConfigured -from scrapy.utils.misc import load_class +from scrapy.utils.misc import load_object from scrapy.conf import settings class ScrapyProcessProtocol(protocol.ProcessProtocol): @@ -95,7 +95,7 @@ class ClusterWorker(pb.Root): self.running = {} # dict of domain->ScrapyProcessControl self.crawlers = {} # dict of pid->scrapy process remote pb connection self.starttime = datetime.datetime.utcnow() - self.prerun_hooks = [load_class(f) for f in settings.getlist('CLUSTER_WORKER_PRERUN_HOOKS', [])] + self.prerun_hooks = [load_object(f) for f in settings.getlist('CLUSTER_WORKER_PRERUN_HOOKS', [])] port = settings.getint('CLUSTER_WORKER_PORT') factory = pb.PBServerFactory(self, unsafeTracebacks=True) scrapyengine.listenTCP(port, factory) diff --git a/scrapy/trunk/scrapy/core/downloader/middleware.py b/scrapy/trunk/scrapy/core/downloader/middleware.py index e7f114c7a..28b4b633d 100644 --- a/scrapy/trunk/scrapy/core/downloader/middleware.py +++ b/scrapy/trunk/scrapy/core/downloader/middleware.py @@ -10,7 +10,7 @@ from scrapy.core import signals from scrapy import log from scrapy.http import Request, Response from scrapy.core.exceptions import NotConfigured -from scrapy.utils.misc import load_class +from scrapy.utils.misc import load_object from scrapy.utils.defer import mustbe_deferred from scrapy.core.downloader.handlers import download_any from scrapy.conf import settings @@ -38,7 +38,7 @@ class DownloaderMiddlewareManager(object): """ mws = [] for mwpath in settings.getlist('DOWNLOADER_MIDDLEWARES') or (): - cls = load_class(mwpath) + cls = load_object(mwpath) if cls: try: mw = cls() diff --git a/scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py b/scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py index 32550ad3f..c9b5c2cea 100644 --- a/scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py +++ b/scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py @@ -8,7 +8,7 @@ from os.path import abspath, dirname, join from mimetypes import MimeTypes from scrapy.http import Response -from scrapy.utils.misc import load_class +from scrapy.utils.misc import load_object from scrapy.utils.python import isbinarytext from scrapy.conf import settings @@ -27,7 +27,7 @@ class ResponseTypes(object): mimefile = join(abspath(dirname(__file__)), 'mime.types') self.mimetypes = MimeTypes([mimefile]) for mimetype, cls in self.CLASSES.iteritems(): - self.classes[mimetype] = load_class(cls) + self.classes[mimetype] = load_object(cls) def from_mimetype(self, mimetype): """Return the most appropiate Response class for the given mimetype""" diff --git a/scrapy/trunk/scrapy/core/manager.py b/scrapy/trunk/scrapy/core/manager.py index a699c5346..dc03da567 100644 --- a/scrapy/trunk/scrapy/core/manager.py +++ b/scrapy/trunk/scrapy/core/manager.py @@ -5,7 +5,7 @@ from scrapy import log from scrapy.http import Request from scrapy.core.engine import scrapyengine from scrapy.spider import spiders -from scrapy.utils.misc import load_class +from scrapy.utils.misc import load_object from scrapy.utils.url import is_url from scrapy.conf import settings @@ -27,11 +27,11 @@ class ExecutionManager(object): extensions.load() log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys())) - scheduler = load_class(settings['SCHEDULER'])() + scheduler = load_object(settings['SCHEDULER'])() scrapyengine.configure(scheduler=scheduler) - self.prioritizer_class = load_class(settings['PRIORITIZER']) + self.prioritizer_class = load_object(settings['PRIORITIZER']) requests = self._parse_args(args) self.priorities = self.prioritizer_class(requests.keys()) diff --git a/scrapy/trunk/scrapy/extension/__init__.py b/scrapy/trunk/scrapy/extension/__init__.py index dc96e1194..a9d81f9c8 100644 --- a/scrapy/trunk/scrapy/extension/__init__.py +++ b/scrapy/trunk/scrapy/extension/__init__.py @@ -4,7 +4,7 @@ keeping track of all enabled extensions. It also contains an instantiated ExtensionManager (extensions) to be used as singleton. """ from scrapy.core.exceptions import NotConfigured -from scrapy.utils.misc import load_class +from scrapy.utils.misc import load_object from scrapy import log from scrapy.conf import settings @@ -25,7 +25,7 @@ class ExtensionManager(object): self.disabled.clear() for extension_path in settings.getlist('EXTENSIONS'): try: - cls = load_class(extension_path) + cls = load_object(extension_path) self.enabled[cls.__name__] = cls() except NotConfigured, e: self.disabled[cls.__name__] = extension_path diff --git a/scrapy/trunk/scrapy/item/pipeline.py b/scrapy/trunk/scrapy/item/pipeline.py index fd86fee4b..6dced5ae7 100644 --- a/scrapy/trunk/scrapy/item/pipeline.py +++ b/scrapy/trunk/scrapy/item/pipeline.py @@ -1,7 +1,7 @@ from scrapy import log from scrapy.core.exceptions import DropItem, NotConfigured from scrapy.item import ScrapedItem -from scrapy.utils.misc import load_class +from scrapy.utils.misc import load_object from scrapy.utils.defer import defer_succeed, mustbe_deferred from scrapy.conf import settings @@ -18,7 +18,7 @@ class ItemPipelineManager(object): Load pipelines stages defined in settings module """ for stage in settings.getlist('ITEM_PIPELINES') or (): - cls = load_class(stage) + cls = load_object(stage) if cls: try: stageinstance = cls() diff --git a/scrapy/trunk/scrapy/spider/manager.py b/scrapy/trunk/scrapy/spider/manager.py index 6e2c1ee89..522d29689 100644 --- a/scrapy/trunk/scrapy/spider/manager.py +++ b/scrapy/trunk/scrapy/spider/manager.py @@ -12,7 +12,7 @@ from scrapy.spider.models import ISpider from scrapy import log from scrapy.conf import settings from scrapy.utils.url import url_is_from_spider -from scrapy.utils.misc import load_class +from scrapy.utils.misc import load_object class SpiderManager(object): """Spider locator and manager""" @@ -54,7 +54,7 @@ class SpiderManager(object): if not spider: # create a custom spider spiderclassname = settings.get('DEFAULT_SPIDER') if spiderclassname: - spider = load_class(spiderclassname)(domain) + spider = load_object(spiderclassname)(domain) self.add_spider(spider) return spider diff --git a/scrapy/trunk/scrapy/spider/middleware.py b/scrapy/trunk/scrapy/spider/middleware.py index 19d0c4312..7dd1a61f6 100644 --- a/scrapy/trunk/scrapy/spider/middleware.py +++ b/scrapy/trunk/scrapy/spider/middleware.py @@ -8,7 +8,7 @@ docs/topics/spider-middleware.rst from scrapy import log from scrapy.core.exceptions import NotConfigured -from scrapy.utils.misc import load_class +from scrapy.utils.misc import load_object from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.conf import settings @@ -38,7 +38,7 @@ class SpiderMiddlewareManager(object): """Load middleware defined in settings module""" mws = [] for mwpath in settings.getlist('SPIDER_MIDDLEWARES') or (): - cls = load_class(mwpath) + cls = load_object(mwpath) if cls: try: mw = cls() diff --git a/scrapy/trunk/scrapy/utils/misc.py b/scrapy/trunk/scrapy/utils/misc.py index 23d16de3f..8886f1afd 100644 --- a/scrapy/trunk/scrapy/utils/misc.py +++ b/scrapy/trunk/scrapy/utils/misc.py @@ -60,24 +60,31 @@ def stats_getpath(dict_, path, default=None): return default return dict_ -def load_class(class_path): - """Load a class given its absolute class path, and return it without - instantiating it""" +def load_object(path): + """Load an object given its absolute object path, and return it. + + object can be a class, function, variable o instance. + path ie: 'scrapy.contrib.downloadermiddelware.redirect.RedirectMiddleware' + """ + try: - dot = class_path.rindex('.') + dot = path.rindex('.') except ValueError: - raise UsageError, '%s isn\'t a module' % class_path - module, classname = class_path[:dot], class_path[dot+1:] + raise UsageError, '%s isn\'t a module' % path + + module, name = path[:dot], path[dot+1:] try: mod = __import__(module, {}, {}, ['']) except ImportError, e: raise UsageError, 'Error importing %s: "%s"' % (module, e) - try: - cls = getattr(mod, classname) - except AttributeError: - raise UsageError, 'module "%s" does not define a "%s" class' % (module, classname) - return cls + try: + obj = getattr(mod, name) + except AttributeError: + raise UsageError, 'module "%s" does not define any object named "%s"' % (module, name) + + return obj +load_class = load_object # backwards compatibility, but isnt going to be available for too long. def extract_regex(regex, text, encoding): """Extract a list of unicode strings from the given text/encoding using the following policies: