Merge pull request #738 from chekunkov/downloader_setting

Downloader setting

docs/topics/settings.rst

@@ -279,6 +279,15 @@ Default: ``True``
 
 Whether to enable DNS in-memory cache.
 
+.. setting:: DOWNLOADER
+
+DOWNLOADER
+----------
+
+Default: ``'scrapy.core.downloader.Downloader'``
+
+The downloader to use for crawling.
+
 .. setting:: DOWNLOADER_MIDDLEWARES
 
 DOWNLOADER_MIDDLEWARES

scrapy/core/engine.py

@@ -60,7 +60,8 @@ class ExecutionEngine(object):
         self.running = False
         self.paused = False
         self.scheduler_cls = load_object(self.settings['SCHEDULER'])
-        self.downloader = Downloader(crawler)
+        downloader_cls = load_object(self.settings['DOWNLOADER'])
+        self.downloader = downloader_cls(crawler)
         self.scraper = Scraper(crawler)
         self._concurrent_spiders = self.settings.getint('CONCURRENT_SPIDERS', 1)
         if self._concurrent_spiders != 1:

scrapy/settings/default_settings.py

@@ -66,6 +66,8 @@ DOWNLOAD_HANDLERS_BASE = {
 
 DOWNLOAD_TIMEOUT = 180      # 3mins
 
+DOWNLOADER = 'scrapy.core.downloader.Downloader'
+
 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
 DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'