RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

allow to override httpclientfactory

This commit is contained in:
Daniel Grana 2009-09-10 16:28:08 -03:00
parent 269724a2b7
commit 734464825b
2 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/conf/default_settings.py
View File

@ -49,6 +49,8 @@ DOWNLOAD_TIMEOUT = 180 # 3mins
DOWNLOADER_DEBUG = False
DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
DOWNLOADER_MIDDLEWARES = {}
DOWNLOADER_MIDDLEWARES_BASE = {

5
scrapy/core/downloader/handlers.py
View File

@ -18,11 +18,12 @@ from scrapy.core.exceptions import NotSupported
from scrapy.utils.defer import defer_succeed
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.signal import send_catch_log
from scrapy.utils.misc import load_object
from scrapy.core.downloader.responsetypes import responsetypes
from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
from scrapy.conf import settings
HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
ssl_supported = 'ssl' in optional_features
@ -44,7 +45,7 @@ def create_factory(request, spider):
"""Return HTTPClientFactory for the given Request"""
url = urlparse.urldefrag(request.url)[0]
timeout = getattr(spider, "download_timeout", None) or default_timeout
factory = ScrapyHTTPClientFactory.from_request(request, timeout)
factory = HTTPClientFactory.from_request(request, timeout)
def _create_response(body):
body = body or ''