diff --git a/scrapy/__init__.py b/scrapy/__init__.py index a02f1644c..17bf64202 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -43,3 +43,8 @@ except ImportError: pass else: optional_features.add('django') + +from twisted import version as _txv +twisted_version = (_txv.major, _txv.minor, _txv.micro) +if twisted_version >= (11, 1, 0): + optional_features.add('http11') diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 9f3799c59..8d05907f1 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,25 +1,11 @@ -"""Download handlers for http and https schemes -""" -from twisted.internet import reactor -from scrapy.utils.misc import load_object +from scrapy import optional_features +from .http10 import HTTP10DownloadHandler + +if 'http11' in optional_features: + from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler +else: + HTTPDownloadHandler = HTTP10DownloadHandler -class HttpDownloadHandler(object): - - def __init__(self, settings): - self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) - self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - - def download_request(self, request, spider): - """Return a deferred for the HTTP download""" - factory = self.HTTPClientFactory(request) - self._connect(factory) - return factory.deferred - - def _connect(self, factory): - host, port = factory.host, factory.port - if factory.scheme == 'https': - return reactor.connectSSL(host, port, factory, - self.ClientContextFactory()) - else: - return reactor.connectTCP(host, port, factory) +# backwards compatibility +HttpDownloadHandler = HTTP10DownloadHandler diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py new file mode 100644 index 000000000..11b2acdae --- /dev/null +++ b/scrapy/core/downloader/handlers/http10.py @@ -0,0 +1,25 @@ +"""Download handlers for http and https schemes +""" +from twisted.internet import reactor +from scrapy.utils.misc import load_object + + +class HTTP10DownloadHandler(object): + + def __init__(self, settings): + self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) + self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + + def download_request(self, request, spider): + """Return a deferred for the HTTP download""" + factory = self.HTTPClientFactory(request) + self._connect(factory) + return factory.deferred + + def _connect(self, factory): + host, port = factory.host, factory.port + if factory.scheme == 'https': + return reactor.connectSSL(host, port, factory, + self.ClientContextFactory()) + else: + return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 6f233a546..456f48054 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -21,7 +21,7 @@ from scrapy import log -class Http11DownloadHandler(object): +class HTTP11DownloadHandler(object): def __init__(self, settings): self._pool = HTTPConnectionPool(reactor, persistent=True) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e18362b13..94fac60d6 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -53,8 +53,8 @@ DOWNLOAD_DELAY = 0 DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', - 'http': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler', - 'https': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler', + 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', + 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler', } diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 1307f57ad..472817777 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -12,8 +12,9 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers.file import FileDownloadHandler -from scrapy.core.downloader.handlers.http import HttpDownloadHandler -from scrapy.core.downloader.handlers.http11 import Http11DownloadHandler +from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler +from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler +from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spider import BaseSpider from scrapy.http import Request @@ -48,7 +49,7 @@ class FileTestCase(unittest.TestCase): class HttpTestCase(unittest.TestCase): - download_handler_cls = HttpDownloadHandler + download_handler_cls = HTTPDownloadHandler def setUp(self): name = self.mktemp() @@ -140,11 +141,20 @@ class HttpTestCase(unittest.TestCase): return d +class DeprecatedHttpTestCase(HttpTestCase): + """HTTP 1.0 test case""" + download_handler_cls = HttpDownloadHandler + + +class Http10TestCase(HttpTestCase): + """HTTP 1.0 test case""" + download_handler_cls = HTTP10DownloadHandler + + class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls = Http11DownloadHandler - - if twisted.__version__.split('.') < (11, 1, 0): + download_handler_cls = HTTP11DownloadHandler + if 'http11' not in optional_features: skip = 'HTTP1.1 not supported in twisted < 11.1.0' @@ -159,8 +169,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): - - download_handler_cls = HttpDownloadHandler + download_handler_cls = HTTPDownloadHandler def setUp(self): site = server.Site(UriResource(), timeout=None) @@ -199,10 +208,18 @@ class HttpProxyTestCase(unittest.TestCase): return self.download_request(request, BaseSpider('foo')).addCallback(_test) -class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls = Http11DownloadHandler +class DeprecatedHttpProxyTestCase(unittest.TestCase): + """Old deprecated reference to http10 downloader handler""" + download_handler_cls = HttpDownloadHandler - if twisted.__version__.split('.') < (11, 1, 0): + +class Http10ProxyTestCase(HttpProxyTestCase): + download_handler_cls = HTTP10DownloadHandler + + +class Http11ProxyTestCase(HttpProxyTestCase): + download_handler_cls = HTTP11DownloadHandler + if 'http11' not in optional_features: skip = 'HTTP1.1 not supported in twisted < 11.1.0' diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py index e184a6735..1ac4e0108 100644 --- a/scrapy/xlib/tx/__init__.py +++ b/scrapy/xlib/tx/__init__.py @@ -1,9 +1,8 @@ -import twisted -txver = twisted.__version__.split('.') -if txver > (13, 0, 0): +from scrapy import twisted_version +if twisted_version > (13, 0, 0): from twisted.web import client from twisted.internet import endpoints -if txver >= (11, 1, 0): +if twisted_version >= (11, 1, 0): from . import client, endpoints else: from scrapy.exceptions import NotSupported