diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 035e12c82..f14c2c000 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -279,6 +279,15 @@ Default: ``True`` Whether to enable DNS in-memory cache. +.. setting:: DOWNLOADER + +DOWNLOADER +--------- + +Default: ``'scrapy.core.downloader.Downloader'`` + +The downloader to use for crawling. + .. setting:: DOWNLOADER_MIDDLEWARES DOWNLOADER_MIDDLEWARES diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1da2dba03..717b34764 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -60,7 +60,8 @@ class ExecutionEngine(object): self.running = False self.paused = False self.scheduler_cls = load_object(self.settings['SCHEDULER']) - self.downloader = Downloader(crawler) + downloader_cls = load_object(self.settings['DOWNLOADER']) + self.downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._concurrent_spiders = self.settings.getint('CONCURRENT_SPIDERS', 1) if self._concurrent_spiders != 1: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 17957e5e4..4be551d72 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -66,6 +66,8 @@ DOWNLOAD_HANDLERS_BASE = { DOWNLOAD_TIMEOUT = 180 # 3mins +DOWNLOADER = 'scrapy.core.downloader.Downloader' + DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'