From 2ce43ebbec5b16ea3d56ece8c6e34e2b47de8c22 Mon Sep 17 00:00:00 2001
From: pablo <none@none>
Date: Mon, 6 Jul 2009 01:07:45 -0300
Subject: [PATCH] made downloader/scheduler/spider middlewares code more
 consistent, added enabled/disabled/loaded informational attributes to all of
 them

---
 scrapy/core/downloader/middleware.py | 27 +++++++++++++++------------
 scrapy/core/scheduler/middleware.py  | 28 ++++++++++++++++------------
 scrapy/spider/middleware.py          | 24 ++++++++++++++----------
 3 files changed, 45 insertions(+), 34 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 2a2559967..4c97b0d26 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -19,6 +19,8 @@ class DownloaderMiddlewareManager(object):
 
     def __init__(self):
         self.loaded = False
+        self.enabled = {}
+        self.disabled = {}
         self.request_middleware = []
         self.response_middleware = []
         self.exception_middleware = []
@@ -33,21 +35,22 @@ class DownloaderMiddlewareManager(object):
             self.exception_middleware.insert(0, mw.process_exception)
 
     def load(self):
-        """Load middleware defined in settings module
-        """
-        mws = []
+        """Load middleware defined in settings module"""
         mwlist = build_middleware_list(settings['DOWNLOADER_MIDDLEWARES_BASE'],
                                        settings['DOWNLOADER_MIDDLEWARES'])
+        self.enabled.clear()
+        self.disabled.clear()
         for mwpath in mwlist:
-            cls = load_object(mwpath)
-            if cls:
-                try:
-                    mw = cls()
-                    self._add_middleware(mw)
-                    mws.append(mw)
-                except NotConfigured:
-                    pass
-        log.msg("Enabled downloader middlewares: %s" % ", ".join([type(m).__name__ for m in mws]),
+            try:
+                cls = load_object(mwpath)
+                mw = cls()
+                self.enabled[cls.__name__] = mw
+                self._add_middleware(mw)
+            except NotConfigured, e:
+                self.disabled[cls.__name__] = mwpath
+                if e.args:
+                    log.msg(e)
+        log.msg("Enabled downloader middlewares: %s" % ", ".join(self.enabled.keys()), \
             level=log.DEBUG)
         self.loaded = True
 
diff --git a/scrapy/core/scheduler/middleware.py b/scrapy/core/scheduler/middleware.py
index 89a60ba59..799af905f 100644
--- a/scrapy/core/scheduler/middleware.py
+++ b/scrapy/core/scheduler/middleware.py
@@ -19,27 +19,31 @@ from scrapy.conf import settings
 class SchedulerMiddlewareManager(object):
 
     def __init__(self):
+        self.loaded = False
+        self.enabled = {}
+        self.disabled = {}
         self.mw_cbs = defaultdict(list)
         self.load()
 
     def load(self):
         """Load middleware defined in settings module"""
-        self.mws = []
         mwlist = build_middleware_list(settings['SCHEDULER_MIDDLEWARES_BASE'],
                                        settings['SCHEDULER_MIDDLEWARES'])
+        self.enabled.clear()
+        self.disabled.clear()
         for mwpath in mwlist:
-            cls = load_object(mwpath)
-            if cls:
-                try:
-                    mw = cls()
-                except NotConfigured:
-                    pass
-                else:
-                    self._add_middleware(mw)
-                    self.mws.append(mw)
-
-        log.msg("Enabled scheduler middlewares: %s" % ", ".join([type(m).__name__ for m in self.mws]),
+            try:
+                cls = load_object(mwpath)
+                mw = cls()
+                self.enabled[cls.__name__] = mw
+                self._add_middleware(mw)
+            except NotConfigured, e:
+                self.disabled[cls.__name__] = mwpath
+                if e.args:
+                    log.msg(e)
+        log.msg("Enabled scheduler middlewares: %s" % ", ".join(self.enabled.keys()), \
             level=log.DEBUG)
+        self.loaded = True
 
     def _add_middleware(self, mw):
         for name in ('enqueue_request', 'open_domain', 'close_domain'):
diff --git a/scrapy/spider/middleware.py b/scrapy/spider/middleware.py
index 0f76606ee..4df7599f3 100644
--- a/scrapy/spider/middleware.py
+++ b/scrapy/spider/middleware.py
@@ -20,6 +20,8 @@ def _isiterable(possible_iterator):
 class SpiderMiddlewareManager(object):
     def __init__(self):
         self.loaded = False
+        self.enabled = {}
+        self.disabled = {}
         self.spider_middleware = []
         self.result_middleware = []
         self.exception_middleware = []
@@ -35,19 +37,21 @@ class SpiderMiddlewareManager(object):
 
     def load(self):
         """Load middleware defined in settings module"""
-        mws = []
         mwlist = build_middleware_list(settings['SPIDER_MIDDLEWARES_BASE'],
                                        settings['SPIDER_MIDDLEWARES'])
+        self.enabled.clear()
+        self.disabled.clear()
         for mwpath in mwlist:
-            cls = load_object(mwpath)
-            if cls:
-                try:
-                    mw = cls()
-                    self._add_middleware(mw)
-                    mws.append(mw)
-                except NotConfigured:
-                    pass
-        log.msg("Enabled spider middlewares: %s" % ", ".join([type(m).__name__ for m in mws]),
+            try:
+                cls = load_object(mwpath)
+                mw = cls()
+                self.enabled[cls.__name__] = mw
+                self._add_middleware(mw)
+            except NotConfigured, e:
+                self.disabled[cls.__name__] = mwpath
+                if e.args:
+                    log.msg(e)
+        log.msg("Enabled spider middlewares: %s" % ", ".join(self.enabled.keys()), \
             level=log.DEBUG)
         self.loaded = True