diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index bc5cd742e..0b55d32fa 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -24,6 +24,7 @@ class DownloadHandlers(object): crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) for scheme, clspath in six.iteritems(handlers): self._schemes[scheme] = clspath + self._load_handler(scheme, skip_lazy=True) crawler.signals.connect(self._close, signals.engine_stopped) @@ -39,9 +40,14 @@ class DownloadHandlers(object): self._notconfigured[scheme] = 'no handler available for that scheme' return None + return self._load_handler(scheme) + + def _load_handler(self, scheme, skip_lazy=False): path = self._schemes[scheme] try: dhcls = load_object(path) + if skip_lazy and getattr(dhcls, 'lazy', True): + return None dh = dhcls(self._crawler.settings) except NotConfigured as ex: self._notconfigured[scheme] = str(ex) @@ -49,12 +55,12 @@ class DownloadHandlers(object): except Exception as ex: logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"', {"clspath": path, "scheme": scheme}, - exc_info=True, extra={'crawler': self._crawler}) + exc_info=True, extra={'crawler': self._crawler}) self._notconfigured[scheme] = str(ex) return None else: self._handlers[scheme] = dh - return self._handlers[scheme] + return dh def download_request(self, request, spider): scheme = urlparse_cached(request).scheme diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index d102f2b73..ad25beb3b 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -6,6 +6,8 @@ from scrapy.utils.decorators import defers class DataURIDownloadHandler(object): + lazy = False + def __init__(self, settings): super(DataURIDownloadHandler, self).__init__() diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 9346ce08d..23f25d28d 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -2,7 +2,9 @@ from w3lib.url import file_uri_to_path from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers + class FileDownloadHandler(object): + lazy = False def __init__(self, settings): pass diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 933bc7e8d..c342d4ab1 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -60,7 +60,10 @@ class ReceivedDataProtocol(Protocol): self.body.close() if self.filename else self.body.seek(0) _CODE_RE = re.compile("\d+") + + class FTPDownloadHandler(object): + lazy = False CODE_MAPPING = { "550": 404, diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 0322bbe49..d875fb1e4 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -6,6 +6,7 @@ from scrapy.utils.python import to_unicode class HTTP10DownloadHandler(object): + lazy = False def __init__(self, settings): self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 038db7b47..0673188a1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler(object): + lazy = False def __init__(self, settings): self._pool = HTTPConnectionPool(reactor, persistent=True) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2f8973054..0d0829793 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -41,13 +41,23 @@ from scrapy.exceptions import NotConfigured from tests.mockserver import MockServer, ssl_context_factory, Echo from tests.spiders import SingleRequestSpider + class DummyDH(object): + lazy = False + + def __init__(self, crawler): + pass + + +class DummyLazyDH(object): + # Default is lazy for backwards compatibility def __init__(self, crawler): pass class OffDH(object): + lazy = False def __init__(self, crawler): raise NotConfigured @@ -60,8 +70,6 @@ class LoadTestCase(unittest.TestCase): crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) - for scheme in handlers: # force load handlers - dh._get_handler(scheme) self.assertIn('scheme', dh._handlers) self.assertNotIn('scheme', dh._notconfigured) @@ -70,8 +78,6 @@ class LoadTestCase(unittest.TestCase): crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) - for scheme in handlers: # force load handlers - dh._get_handler(scheme) self.assertNotIn('scheme', dh._handlers) self.assertIn('scheme', dh._notconfigured) @@ -80,11 +86,22 @@ class LoadTestCase(unittest.TestCase): crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertNotIn('scheme', dh._schemes) - for scheme in handlers: # force load handlers + for scheme in handlers: # force load handlers dh._get_handler(scheme) self.assertNotIn('scheme', dh._handlers) self.assertIn('scheme', dh._notconfigured) + def test_lazy_handlers(self): + handlers = {'scheme': 'tests.test_downloader_handlers.DummyLazyDH'} + crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + dh = DownloadHandlers(crawler) + self.assertIn('scheme', dh._schemes) + self.assertNotIn('scheme', dh._handlers) + for scheme in handlers: # force load lazy handler + dh._get_handler(scheme) + self.assertIn('scheme', dh._handlers) + self.assertNotIn('scheme', dh._notconfigured) + class FileTestCase(unittest.TestCase):