diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 50395d3c8..bd6979b23 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -61,7 +61,7 @@ class Scraper(object): def __init__(self, engine, settings): self.sites = {} - self.spidermw = SpiderMiddlewareManager() + self.spidermw = SpiderMiddlewareManager.from_settings(settings) itemproc_cls = load_object(settings['ITEM_PROCESSOR']) self.itemproc = itemproc_cls.from_settings(settings) self.concurrent_items = settings.getint('CONCURRENT_ITEMS') diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 38f947045..0cb79b065 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -1,65 +1,40 @@ """ -This module implements the Spider Middleware manager. For more information see -the Spider Middleware doc in: - -docs/topics/spider-middleware.rst +Spider Middleware manager +See documentation in docs/topics/spider-middleware.rst """ -from scrapy import log from twisted.python.failure import Failure -from scrapy.exceptions import NotConfigured -from scrapy.utils.misc import load_object -from scrapy.utils.conf import build_component_list +from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred -from scrapy.conf import settings +from scrapy.utils.conf import build_component_list def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') -class SpiderMiddlewareManager(object): - def __init__(self): - self.loaded = False - self.enabled = {} - self.disabled = {} - self.spider_middleware = [] - self.result_middleware = [] - self.exception_middleware = [] - self.load() +class SpiderMiddlewareManager(MiddlewareManager): + + component_name = 'spider middleware' + + @classmethod + def _get_mwlist_from_settings(cls, settings): + return build_component_list(settings['SPIDER_MIDDLEWARES_BASE'], \ + settings['SPIDER_MIDDLEWARES']) def _add_middleware(self, mw): + super(SpiderMiddlewareManager, self)._add_middleware(mw) if hasattr(mw, 'process_spider_input'): - self.spider_middleware.append(mw.process_spider_input) + self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_spider_output'): - self.result_middleware.insert(0, mw.process_spider_output) + self.methods['process_spider_output'].append(mw.process_spider_output) if hasattr(mw, 'process_spider_exception'): - self.exception_middleware.insert(0, mw.process_spider_exception) - - def load(self): - """Load middleware defined in settings module""" - mwlist = build_component_list(settings['SPIDER_MIDDLEWARES_BASE'], \ - settings['SPIDER_MIDDLEWARES']) - self.enabled.clear() - self.disabled.clear() - for mwpath in mwlist: - try: - cls = load_object(mwpath) - mw = cls() - self.enabled[cls.__name__] = mw - self._add_middleware(mw) - except NotConfigured, e: - self.disabled[cls.__name__] = mwpath - if e.args: - log.msg(e) - log.msg("Enabled spider middlewares: %s" % ", ".join(self.enabled.keys()), \ - level=log.DEBUG) - self.loaded = True + self.methods['process_spider_exception'].append(mw.process_spider_exception) def scrape_response(self, scrape_func, response, request, spider): fname = lambda f:'%s.%s' % (f.im_self.__class__.__name__, f.im_func.__name__) def process_spider_input(response): - for method in self.spider_middleware: + for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) assert result is None, \ @@ -72,7 +47,7 @@ class SpiderMiddlewareManager(object): def process_spider_exception(_failure): exception = _failure.value - for method in self.exception_middleware: + for method in self.methods['process_spider_exception']: result = method(response=response, exception=exception, spider=spider) assert result is None or _isiterable(result), \ 'Middleware %s must returns None, or an iterable object, got %s ' % \ @@ -82,7 +57,7 @@ class SpiderMiddlewareManager(object): return _failure def process_spider_output(result): - for method in self.result_middleware: + for method in self.methods['process_spider_output']: result = method(response=response, result=result, spider=spider) assert _isiterable(result), \ 'Middleware %s must returns an iterable object, got %s ' % \