diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 531862d9d..12dda4322 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -380,6 +380,15 @@ A dict containing the request download handlers enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`DOWNLOAD_HANDLERS` instead. +If you want to disable any of the above download handlers you must define them +in your project's :setting:`DOWNLOAD_HANDLERS` setting and assign `None` +as their value. For example, if you want to disable the file download +handler:: + + DOWNLOAD_HANDLERS = { + 'file': None, + } + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index b743220c9..ff82bffd5 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -15,6 +15,10 @@ class DownloadHandlers(object): handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE') handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {})) for scheme, clspath in handlers.iteritems(): + # Allow to disable a handler just like any other + # component (extension, middleware, etc). + if clspath is None: + continue cls = load_object(clspath) try: dh = cls(crawler.settings) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 537013434..a4e54ed76 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -15,6 +15,7 @@ from twisted.protocols.ftp import FTPClient, ConnectionLost from w3lib.url import path_to_file_uri from scrapy import twisted_version +from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler @@ -26,6 +27,41 @@ from scrapy.spider import Spider from scrapy.http import Request from scrapy.settings import Settings from scrapy import optional_features +from scrapy.utils.test import get_crawler +from scrapy.exceptions import NotConfigured + + +class DummyDH(object): + + def __init__(self, crawler): + pass + + +class OffDH(object): + + def __init__(self, crawler): + raise NotConfigured + + +class LoadTestCase(unittest.TestCase): + + def test_enabled_handler(self): + handlers = {'scheme': 'scrapy.tests.test_downloader_handlers.DummyDH'} + dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + self.assertIn('scheme', dh._handlers) + self.assertNotIn('scheme', dh._notconfigured) + + def test_not_configured_handler(self): + handlers = {'scheme': 'scrapy.tests.test_downloader_handlers.OffDH'} + dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + self.assertNotIn('scheme', dh._handlers) + self.assertIn('scheme', dh._notconfigured) + + def test_disabled_handler(self): + handlers = {'scheme': None} + dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + self.assertNotIn('scheme', dh._handlers) + self.assertNotIn('scheme', dh._notconfigured) class FileTestCase(unittest.TestCase):