From 167211ffb0e7fa7756483aa9f20e1ab5589c7c4a Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Mon, 20 Aug 2018 15:54:04 -0300 Subject: [PATCH] Default is lazy, load_object exception handling, code improvements --- scrapy/core/downloader/handlers/__init__.py | 22 +++++++++------------ scrapy/core/downloader/handlers/datauri.py | 2 ++ scrapy/core/downloader/handlers/file.py | 2 ++ scrapy/core/downloader/handlers/ftp.py | 3 +++ scrapy/core/downloader/handlers/http10.py | 1 + scrapy/core/downloader/handlers/http11.py | 1 + scrapy/core/downloader/handlers/s3.py | 1 - tests/test_downloader_handlers.py | 4 +++- 8 files changed, 21 insertions(+), 15 deletions(-) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index ebe6f5b78..0b55d32fa 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -24,13 +24,7 @@ class DownloadHandlers(object): crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) for scheme, clspath in six.iteritems(handlers): self._schemes[scheme] = clspath - for scheme in self._schemes: - path = self._schemes[scheme] - dhcls = load_object(path) - lazy = getattr(dhcls, 'lazy', False) - if lazy: - continue - self._load_handler(scheme, dhcls) + self._load_handler(scheme, skip_lazy=True) crawler.signals.connect(self._close, signals.engine_stopped) @@ -46,13 +40,14 @@ class DownloadHandlers(object): self._notconfigured[scheme] = 'no handler available for that scheme' return None - path = self._schemes[scheme] - dhcls = load_object(path) - self._load_handler(scheme, dhcls) - return self._handlers[scheme] + return self._load_handler(scheme) - def _load_handler(self, scheme, dhcls): + def _load_handler(self, scheme, skip_lazy=False): + path = self._schemes[scheme] try: + dhcls = load_object(path) + if skip_lazy and getattr(dhcls, 'lazy', True): + return None dh = dhcls(self._crawler.settings) except NotConfigured as ex: self._notconfigured[scheme] = str(ex) @@ -60,11 +55,12 @@ class DownloadHandlers(object): except Exception as ex: logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"', {"clspath": path, "scheme": scheme}, - exc_info=True, extra={'crawler': self._crawler}) + exc_info=True, extra={'crawler': self._crawler}) self._notconfigured[scheme] = str(ex) return None else: self._handlers[scheme] = dh + return dh def download_request(self, request, spider): scheme = urlparse_cached(request).scheme diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index d102f2b73..ad25beb3b 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -6,6 +6,8 @@ from scrapy.utils.decorators import defers class DataURIDownloadHandler(object): + lazy = False + def __init__(self, settings): super(DataURIDownloadHandler, self).__init__() diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 9346ce08d..23f25d28d 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -2,7 +2,9 @@ from w3lib.url import file_uri_to_path from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers + class FileDownloadHandler(object): + lazy = False def __init__(self, settings): pass diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 933bc7e8d..c342d4ab1 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -60,7 +60,10 @@ class ReceivedDataProtocol(Protocol): self.body.close() if self.filename else self.body.seek(0) _CODE_RE = re.compile("\d+") + + class FTPDownloadHandler(object): + lazy = False CODE_MAPPING = { "550": 404, diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 0322bbe49..d875fb1e4 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -6,6 +6,7 @@ from scrapy.utils.python import to_unicode class HTTP10DownloadHandler(object): + lazy = False def __init__(self, settings): self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 038db7b47..0673188a1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler(object): + lazy = False def __init__(self, settings): self._pool = HTTPConnectionPool(reactor, persistent=True) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index e723e616d..d8bbdd326 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -31,7 +31,6 @@ def _get_boto_connection(): class S3DownloadHandler(object): - lazy = True def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ httpdownloadhandler=HTTPDownloadHandler, **kw): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 116942ebe..0d0829793 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -43,19 +43,21 @@ from tests.spiders import SingleRequestSpider class DummyDH(object): + lazy = False def __init__(self, crawler): pass class DummyLazyDH(object): - lazy = True + # Default is lazy for backwards compatibility def __init__(self, crawler): pass class OffDH(object): + lazy = False def __init__(self, crawler): raise NotConfigured