diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py index a61a258eb..bd82a97c1 100644 --- a/scrapy/core/downloader/manager.py +++ b/scrapy/core/downloader/manager.py @@ -9,6 +9,7 @@ from twisted.internet import reactor, defer from scrapy.core.exceptions import IgnoreRequest from scrapy.spider import spiders from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.core.downloader.handlers import download_any from scrapy.conf import settings from scrapy.utils.defer import chain_deferred, mustbe_deferred @@ -74,7 +75,7 @@ class Downloader(object): site.active.remove(request) return _ - return self.middleware.download(request, spider).addBoth(_deactivate) + return self.middleware.download(self.enqueue, request, spider).addBoth(_deactivate) def enqueue(self, request, spider): """Enqueue a Request for a effective download from site""" @@ -116,7 +117,7 @@ class Downloader(object): site.downloading.remove(request) self.process_queue(spider) - dwld = mustbe_deferred(request, spider) + dwld = mustbe_deferred(download_any, request, spider) chain_deferred(dwld, deferred) return dwld.addBoth(_finish) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index c3df8cb83..8b87b7cee 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -13,7 +13,6 @@ from scrapy.core.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.utils.defer import mustbe_deferred from scrapy.utils.middleware import build_middleware_list -from scrapy.core.downloader.handlers import download_any from scrapy.conf import settings class DownloaderMiddlewareManager(object): @@ -51,7 +50,7 @@ class DownloaderMiddlewareManager(object): log.msg("Enabled downloader middlewares: %s" % ", ".join([type(m).__name__ for m in mws])) self.loaded = True - def download(self, request, spider): + def download(self, download_func, request, spider): def process_request(request): for method in self.request_middleware: response = method(request=request, spider=spider) @@ -60,7 +59,7 @@ class DownloaderMiddlewareManager(object): (method.im_self.__class__.__name__, response.__class__.__name__) if response: return response - return download_any(request=request, spider=spider) + return download_func(request=request, spider=spider) def process_response(response): assert response is not None, 'Received None in process_response'