Merge pull request #650 from Curita/dlhandler-disable

Allow to disable a downloader handler just like any other component

docs/topics/settings.rst

@@ -380,6 +380,15 @@ A dict containing the request download handlers enabled by default in Scrapy.
 You should never modify this setting in your project, modify
 :setting:`DOWNLOAD_HANDLERS` instead.
 
+If you want to disable any of the above download handlers you must define them
+in your project's :setting:`DOWNLOAD_HANDLERS` setting and assign `None`
+as their value.  For example, if you want to disable the file download
+handler::
+
+    DOWNLOAD_HANDLERS = {
+        'file': None,
+    }
+
 .. setting:: DOWNLOAD_TIMEOUT
 
 DOWNLOAD_TIMEOUT

scrapy/core/downloader/handlers/__init__.py

@@ -15,6 +15,10 @@ class DownloadHandlers(object):
         handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE')
         handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {}))
         for scheme, clspath in handlers.iteritems():
+            # Allow to disable a handler just like any other
+            # component (extension, middleware, etc).
+            if clspath is None:
+                continue
             cls = load_object(clspath)
             try:
                 dh = cls(crawler.settings)

scrapy/tests/test_downloader_handlers.py

@@ -15,6 +15,7 @@ from twisted.protocols.ftp import FTPClient, ConnectionLost
 from w3lib.url import path_to_file_uri
 
 from scrapy import twisted_version
+from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
@@ -26,6 +27,41 @@ from scrapy.spider import Spider
 from scrapy.http import Request
 from scrapy.settings import Settings
 from scrapy import optional_features
+from scrapy.utils.test import get_crawler
+from scrapy.exceptions import NotConfigured
+
+
+class DummyDH(object):
+
+    def __init__(self, crawler):
+        pass
+
+
+class OffDH(object):
+
+    def __init__(self, crawler):
+        raise NotConfigured
+
+
+class LoadTestCase(unittest.TestCase):
+
+    def test_enabled_handler(self):
+        handlers = {'scheme': 'scrapy.tests.test_downloader_handlers.DummyDH'}
+        dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers}))
+        self.assertIn('scheme', dh._handlers)
+        self.assertNotIn('scheme', dh._notconfigured)
+
+    def test_not_configured_handler(self):
+        handlers = {'scheme': 'scrapy.tests.test_downloader_handlers.OffDH'}
+        dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers}))
+        self.assertNotIn('scheme', dh._handlers)
+        self.assertIn('scheme', dh._notconfigured)
+
+    def test_disabled_handler(self):
+        handlers = {'scheme': None}
+        dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers}))
+        self.assertNotIn('scheme', dh._handlers)
+        self.assertNotIn('scheme', dh._notconfigured)
 
 
 class FileTestCase(unittest.TestCase):