utils: renamed load_class function as load_object

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40853

scrapy/trunk/scrapy/command/commands/shell.py

@@ -5,7 +5,7 @@ import scrapy
 from scrapy.command import ScrapyCommand
 from scrapy.spider import spiders
 from scrapy.xpath import XmlXPathSelector, HtmlXPathSelector
-from scrapy.utils.misc import load_class
+from scrapy.utils.misc import load_object
 from scrapy.extension import extensions
 from scrapy.conf import settings
 from scrapy.core.manager import scrapymanager
@@ -85,7 +85,7 @@ class Command(ScrapyCommand):
             return True
 
     def generate_vars(self, url, response):
-        itemcls = load_class(settings['DEFAULT_ITEM_CLASS'])
+        itemcls = load_object(settings['DEFAULT_ITEM_CLASS'])
         item = itemcls()
         self.vars['item'] = item
         if url:

scrapy/trunk/scrapy/contrib/history/middleware.py

@@ -3,7 +3,7 @@ from datetime import datetime
 
 from pydispatch import dispatcher
 
-from scrapy.utils.misc import load_class
+from scrapy.utils.misc import load_object
 from scrapy.core import signals
 from scrapy import log
 from scrapy.core.exceptions import NotConfigured, IgnoreRequest
@@ -16,7 +16,7 @@ class HistoryMiddleware(object):
     MIN_PROCESS_UNCHANGED_DAYS = 12
 
     def __init__(self):
-        historycls = load_class(settings['MEMORYSTORE'])
+        historycls = load_object(settings['MEMORYSTORE'])
         if not historycls:
             raise NotConfigured
         self.historydata = historycls()

scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py

@@ -8,7 +8,7 @@ from scrapy.core import signals
 from scrapy.http import Request
 from scrapy.core.exceptions import NotConfigured
 from scrapy.utils.request import request_fingerprint
-from scrapy.utils.misc import load_class
+from scrapy.utils.misc import load_object
 from scrapy.conf import settings
 from scrapy import log
 
@@ -34,7 +34,7 @@ class DuplicatesFilterMiddleware(object):
         if not clspath:
             raise NotConfigured
 
-        self.filter = load_class(clspath)()
+        self.filter = load_object(clspath)()
         dispatcher.connect(self.filter.open, signals.domain_open)
         dispatcher.connect(self.filter.close, signals.domain_closed)
 

scrapy/trunk/scrapy/contrib_exp/cluster/worker/manager.py

@@ -11,7 +11,7 @@ from twisted.spread import pb
 from scrapy import log
 from scrapy.core.engine import scrapyengine
 from scrapy.core.exceptions import NotConfigured
-from scrapy.utils.misc import load_class
+from scrapy.utils.misc import load_object
 from scrapy.conf import settings
 
 class ScrapyProcessProtocol(protocol.ProcessProtocol):
@@ -95,7 +95,7 @@ class ClusterWorker(pb.Root):
         self.running = {} # dict of domain->ScrapyProcessControl 
         self.crawlers = {} # dict of pid->scrapy process remote pb connection
         self.starttime = datetime.datetime.utcnow()
-        self.prerun_hooks = [load_class(f) for f in settings.getlist('CLUSTER_WORKER_PRERUN_HOOKS', [])]
+        self.prerun_hooks = [load_object(f) for f in settings.getlist('CLUSTER_WORKER_PRERUN_HOOKS', [])]
         port = settings.getint('CLUSTER_WORKER_PORT')
         factory = pb.PBServerFactory(self, unsafeTracebacks=True)
         scrapyengine.listenTCP(port, factory)

scrapy/trunk/scrapy/core/downloader/middleware.py

@@ -10,7 +10,7 @@ from scrapy.core import signals
 from scrapy import log
 from scrapy.http import Request, Response
 from scrapy.core.exceptions import NotConfigured
-from scrapy.utils.misc import load_class
+from scrapy.utils.misc import load_object
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.core.downloader.handlers import download_any
 from scrapy.conf import settings
@@ -38,7 +38,7 @@ class DownloaderMiddlewareManager(object):
         """
         mws = []
         for mwpath in settings.getlist('DOWNLOADER_MIDDLEWARES') or ():
-            cls = load_class(mwpath)
+            cls = load_object(mwpath)
             if cls:
                 try:
                     mw = cls()

scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py

@@ -8,7 +8,7 @@ from os.path import abspath, dirname, join
 from mimetypes import MimeTypes
 
 from scrapy.http import Response
-from scrapy.utils.misc import load_class
+from scrapy.utils.misc import load_object
 from scrapy.utils.python import isbinarytext
 from scrapy.conf import settings
 
@@ -27,7 +27,7 @@ class ResponseTypes(object):
         mimefile = join(abspath(dirname(__file__)), 'mime.types')
         self.mimetypes = MimeTypes([mimefile])
         for mimetype, cls in self.CLASSES.iteritems():
-            self.classes[mimetype] = load_class(cls)
+            self.classes[mimetype] = load_object(cls)
 
     def from_mimetype(self, mimetype):
         """Return the most appropiate Response class for the given mimetype"""

scrapy/trunk/scrapy/core/manager.py

@@ -5,7 +5,7 @@ from scrapy import log
 from scrapy.http import Request
 from scrapy.core.engine import scrapyengine
 from scrapy.spider import spiders
-from scrapy.utils.misc import load_class
+from scrapy.utils.misc import load_object
 from scrapy.utils.url import is_url
 from scrapy.conf import settings
 
@@ -27,11 +27,11 @@ class ExecutionManager(object):
         extensions.load()
         log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys()))
 
-        scheduler = load_class(settings['SCHEDULER'])()
+        scheduler = load_object(settings['SCHEDULER'])()
 
         scrapyengine.configure(scheduler=scheduler)
 
-        self.prioritizer_class = load_class(settings['PRIORITIZER'])
+        self.prioritizer_class = load_object(settings['PRIORITIZER'])
 
         requests = self._parse_args(args)
         self.priorities = self.prioritizer_class(requests.keys())

scrapy/trunk/scrapy/extension/__init__.py

@@ -4,7 +4,7 @@ keeping track of all enabled extensions. It also contains an instantiated
 ExtensionManager (extensions) to be used as singleton.
 """
 from scrapy.core.exceptions import NotConfigured
-from scrapy.utils.misc import load_class
+from scrapy.utils.misc import load_object
 from scrapy import log
 from scrapy.conf import settings
 
@@ -25,7 +25,7 @@ class ExtensionManager(object):
         self.disabled.clear()
         for extension_path in settings.getlist('EXTENSIONS'):
             try:
-                cls = load_class(extension_path)
+                cls = load_object(extension_path)
                 self.enabled[cls.__name__] = cls()
             except NotConfigured, e:
                 self.disabled[cls.__name__] = extension_path

scrapy/trunk/scrapy/item/pipeline.py

@@ -1,7 +1,7 @@
 from scrapy import log
 from scrapy.core.exceptions import DropItem, NotConfigured
 from scrapy.item import ScrapedItem
-from scrapy.utils.misc import load_class
+from scrapy.utils.misc import load_object
 from scrapy.utils.defer import defer_succeed, mustbe_deferred
 from scrapy.conf import settings
 
@@ -18,7 +18,7 @@ class ItemPipelineManager(object):
         Load pipelines stages defined in settings module
         """
         for stage in settings.getlist('ITEM_PIPELINES') or ():
-            cls = load_class(stage)
+            cls = load_object(stage)
             if cls:
                 try:
                     stageinstance = cls()

scrapy/trunk/scrapy/spider/manager.py

@@ -12,7 +12,7 @@ from scrapy.spider.models import ISpider
 from scrapy import log
 from scrapy.conf import settings
 from scrapy.utils.url import url_is_from_spider
-from scrapy.utils.misc import load_class
+from scrapy.utils.misc import load_object
 
 class SpiderManager(object):
     """Spider locator and manager"""
@@ -54,7 +54,7 @@ class SpiderManager(object):
         if not spider:                          # create a custom spider
             spiderclassname = settings.get('DEFAULT_SPIDER')
             if spiderclassname:
-                spider = load_class(spiderclassname)(domain)
+                spider = load_object(spiderclassname)(domain)
                 self.add_spider(spider)
             
         return spider

scrapy/trunk/scrapy/spider/middleware.py

@@ -8,7 +8,7 @@ docs/topics/spider-middleware.rst
 
 from scrapy import log
 from scrapy.core.exceptions import NotConfigured
-from scrapy.utils.misc import load_class
+from scrapy.utils.misc import load_object
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy.conf import settings
 
@@ -38,7 +38,7 @@ class SpiderMiddlewareManager(object):
         """Load middleware defined in settings module"""
         mws = []
         for mwpath in settings.getlist('SPIDER_MIDDLEWARES') or ():
-            cls = load_class(mwpath)
+            cls = load_object(mwpath)
             if cls:
                 try:
                     mw = cls()

scrapy/trunk/scrapy/utils/misc.py

@@ -60,24 +60,31 @@ def stats_getpath(dict_, path, default=None):
             return default
     return dict_
 
-def load_class(class_path):
-    """Load a class given its absolute class path, and return it without
-    instantiating it"""
+def load_object(path):
+    """Load an object given its absolute object path, and return it.
+
+    object can be a class, function, variable o instance.
+    path ie: 'scrapy.contrib.downloadermiddelware.redirect.RedirectMiddleware'
+    """
+
     try:
-        dot = class_path.rindex('.')
+        dot = path.rindex('.')
     except ValueError:
-        raise UsageError, '%s isn\'t a module' % class_path
-    module, classname = class_path[:dot], class_path[dot+1:]
+        raise UsageError, '%s isn\'t a module' % path
+
+    module, name = path[:dot], path[dot+1:]
     try:
         mod = __import__(module, {}, {}, [''])
     except ImportError, e:
         raise UsageError, 'Error importing %s: "%s"' % (module, e)
-    try:
-        cls = getattr(mod, classname)
-    except AttributeError:
-        raise UsageError, 'module "%s" does not define a "%s" class' % (module, classname)
 
-    return cls
+    try:
+        obj = getattr(mod, name)
+    except AttributeError:
+        raise UsageError, 'module "%s" does not define any object named "%s"' % (module, name)
+
+    return obj
+load_class = load_object # backwards compatibility, but isnt going to be available for too long.
 
 def extract_regex(regex, text, encoding):
     """Extract a list of unicode strings from the given text/encoding using the following policies: