RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

made downloader/scheduler/spider middlewares code more consistent, added enabled/disabled/loaded informational attributes to all of them

This commit is contained in:
pablo 2009-07-06 01:07:45 -03:00
parent f467c233b2
commit 2ce43ebbec
3 changed files with 45 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
scrapy/core/downloader/middleware.py
View File

@ -19,6 +19,8 @@ class DownloaderMiddlewareManager(object):
def __init__(self):
self.loaded = False
self.enabled = {}
self.disabled = {}
self.request_middleware = []
self.response_middleware = []
self.exception_middleware = []
@ -33,21 +35,22 @@ class DownloaderMiddlewareManager(object):
self.exception_middleware.insert(0, mw.process_exception)
def load(self):
"""Load middleware defined in settings module
"""
mws = []
"""Load middleware defined in settings module"""
mwlist = build_middleware_list(settings['DOWNLOADER_MIDDLEWARES_BASE'],
settings['DOWNLOADER_MIDDLEWARES'])
self.enabled.clear()
self.disabled.clear()
for mwpath in mwlist:
cls = load_object(mwpath)
if cls:
try:
mw = cls()
self._add_middleware(mw)
mws.append(mw)
except NotConfigured:
pass
log.msg("Enabled downloader middlewares: %s" % ", ".join([type(m).__name__ for m in mws]),
try:
cls = load_object(mwpath)
mw = cls()
self.enabled[cls.__name__] = mw
self._add_middleware(mw)
except NotConfigured, e:
self.disabled[cls.__name__] = mwpath
if e.args:
log.msg(e)
log.msg("Enabled downloader middlewares: %s" % ", ".join(self.enabled.keys()), \
level=log.DEBUG)
self.loaded = True

28
scrapy/core/scheduler/middleware.py
View File

@ -19,27 +19,31 @@ from scrapy.conf import settings
class SchedulerMiddlewareManager(object):
def __init__(self):
self.loaded = False
self.enabled = {}
self.disabled = {}
self.mw_cbs = defaultdict(list)
self.load()
def load(self):
"""Load middleware defined in settings module"""
self.mws = []
mwlist = build_middleware_list(settings['SCHEDULER_MIDDLEWARES_BASE'],
settings['SCHEDULER_MIDDLEWARES'])
self.enabled.clear()
self.disabled.clear()
for mwpath in mwlist:
cls = load_object(mwpath)
if cls:
try:
mw = cls()
except NotConfigured:
pass
else:
self._add_middleware(mw)
self.mws.append(mw)
log.msg("Enabled scheduler middlewares: %s" % ", ".join([type(m).__name__ for m in self.mws]),
try:
cls = load_object(mwpath)
mw = cls()
self.enabled[cls.__name__] = mw
self._add_middleware(mw)
except NotConfigured, e:
self.disabled[cls.__name__] = mwpath
if e.args:
log.msg(e)
log.msg("Enabled scheduler middlewares: %s" % ", ".join(self.enabled.keys()), \
level=log.DEBUG)
self.loaded = True
def _add_middleware(self, mw):
for name in ('enqueue_request', 'open_domain', 'close_domain'):

24
scrapy/spider/middleware.py
View File

@ -20,6 +20,8 @@ def _isiterable(possible_iterator):
class SpiderMiddlewareManager(object):
def __init__(self):
self.loaded = False
self.enabled = {}
self.disabled = {}
self.spider_middleware = []
self.result_middleware = []
self.exception_middleware = []
@ -35,19 +37,21 @@ class SpiderMiddlewareManager(object):
def load(self):
"""Load middleware defined in settings module"""
mws = []
mwlist = build_middleware_list(settings['SPIDER_MIDDLEWARES_BASE'],
settings['SPIDER_MIDDLEWARES'])
self.enabled.clear()
self.disabled.clear()
for mwpath in mwlist:
cls = load_object(mwpath)
if cls:
try:
mw = cls()
self._add_middleware(mw)
mws.append(mw)
except NotConfigured:
pass
log.msg("Enabled spider middlewares: %s" % ", ".join([type(m).__name__ for m in mws]),
try:
cls = load_object(mwpath)
mw = cls()
self.enabled[cls.__name__] = mw
self._add_middleware(mw)
except NotConfigured, e:
self.disabled[cls.__name__] = mwpath
if e.args:
log.msg(e)
log.msg("Enabled spider middlewares: %s" % ", ".join(self.enabled.keys()), \
level=log.DEBUG)
self.loaded = True