From 734464825b61f7853b23bc32df68704df84a38db Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 10 Sep 2009 16:28:08 -0300 Subject: [PATCH] allow to override httpclientfactory --- scrapy/conf/default_settings.py | 2 ++ scrapy/core/downloader/handlers.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 890fb11c6..ce1712fb2 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -49,6 +49,8 @@ DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOADER_DEBUG = False +DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' + DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { diff --git a/scrapy/core/downloader/handlers.py b/scrapy/core/downloader/handlers.py index df5bf1903..016e6677d 100644 --- a/scrapy/core/downloader/handlers.py +++ b/scrapy/core/downloader/handlers.py @@ -18,11 +18,12 @@ from scrapy.core.exceptions import NotSupported from scrapy.utils.defer import defer_succeed from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.signal import send_catch_log +from scrapy.utils.misc import load_object from scrapy.core.downloader.responsetypes import responsetypes -from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory from scrapy.conf import settings +HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) default_timeout = settings.getint('DOWNLOAD_TIMEOUT') ssl_supported = 'ssl' in optional_features @@ -44,7 +45,7 @@ def create_factory(request, spider): """Return HTTPClientFactory for the given Request""" url = urlparse.urldefrag(request.url)[0] timeout = getattr(spider, "download_timeout", None) or default_timeout - factory = ScrapyHTTPClientFactory.from_request(request, timeout) + factory = HTTPClientFactory.from_request(request, timeout) def _create_response(body): body = body or ''