From a1df15b9cea35c3a215778e57d199977fe3da59a Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Wed, 6 Nov 2013 21:21:35 -0400 Subject: [PATCH 1/5] Alow to disable a downloader handler just like any other component. --- scrapy/core/downloader/handlers/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index b743220c9..fba64fd71 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -15,6 +15,10 @@ class DownloadHandlers(object): handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE') handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {})) for scheme, clspath in handlers.iteritems(): + # Allow to disable a handler just like any other + # component (extension, middlware, etc). + if clspath is None: + continue cls = load_object(clspath) try: dh = cls(crawler.settings) From efb86f966d8be992b9af06c78ef820fcdc64a6d7 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Wed, 12 Mar 2014 22:32:48 -0300 Subject: [PATCH 2/5] Tests for loading download handlers --- scrapy/tests/test_downloader_handlers.py | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 537013434..cf7b1df78 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -15,6 +15,7 @@ from twisted.protocols.ftp import FTPClient, ConnectionLost from w3lib.url import path_to_file_uri from scrapy import twisted_version +from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler @@ -26,6 +27,40 @@ from scrapy.spider import Spider from scrapy.http import Request from scrapy.settings import Settings from scrapy import optional_features +from scrapy.utils.test import get_crawler +from scrapy.exceptions import NotConfigured + + +class DummyDH(object): + + def __init__(self, settings): + pass + + +class OffDH(object): + + def __init__(self, crawler): + raise NotConfigured + + +class LoadTestCase(unittest.TestCase): + + def test_enabled_handler(self): + handlers = {'scheme': 'scrapy.tests.test_downloader_handlers.DummyDH'} + dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + self.assertIn('scheme', dh._handlers) + self.assertNotIn('scheme', dh._notconfigured) + + def test_not_configured_handler(self): + handlers = {'scheme': 'scrapy.tests.test_downloader_handlers.OffDH'} + dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + self.assertNotIn('scheme', dh._handlers) + self.assertIn('scheme', dh._notconfigured) + + def test_disabled_handler(self): + handlers = {'scheme': None} + dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + self.assertNotIn('scheme', dh._handlers) class FileTestCase(unittest.TestCase): From 65734594816b158bdf08b93244795b5dcc8626ba Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Wed, 12 Mar 2014 22:33:42 -0300 Subject: [PATCH 3/5] Fix minor typo in DownloaderHandlers comment --- scrapy/core/downloader/handlers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index fba64fd71..ff82bffd5 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -16,7 +16,7 @@ class DownloadHandlers(object): handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {})) for scheme, clspath in handlers.iteritems(): # Allow to disable a handler just like any other - # component (extension, middlware, etc). + # component (extension, middleware, etc). if clspath is None: continue cls = load_object(clspath) From b9e2aad874ae82d8347cdfb44d113cd1bc613bf3 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Wed, 12 Mar 2014 23:21:33 -0300 Subject: [PATCH 4/5] Doc for disabling download handler --- docs/topics/settings.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 531862d9d..12dda4322 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -380,6 +380,15 @@ A dict containing the request download handlers enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`DOWNLOAD_HANDLERS` instead. +If you want to disable any of the above download handlers you must define them +in your project's :setting:`DOWNLOAD_HANDLERS` setting and assign `None` +as their value. For example, if you want to disable the file download +handler:: + + DOWNLOAD_HANDLERS = { + 'file': None, + } + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT From 2177d4feec9de13e23eba89bf52cf884b0731235 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 13 Mar 2014 01:29:10 -0300 Subject: [PATCH 5/5] Minor fixes in LoadTestCase in test_downloader_handlers * same __init__ parameters in both download handlers's mocks * additional assertion in test_disabled_handler --- scrapy/tests/test_downloader_handlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index cf7b1df78..a4e54ed76 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -33,7 +33,7 @@ from scrapy.exceptions import NotConfigured class DummyDH(object): - def __init__(self, settings): + def __init__(self, crawler): pass @@ -61,6 +61,7 @@ class LoadTestCase(unittest.TestCase): handlers = {'scheme': None} dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) self.assertNotIn('scheme', dh._handlers) + self.assertNotIn('scheme', dh._notconfigured) class FileTestCase(unittest.TestCase):