improve twisted version tests and enable HTTP11 handler by default if supported

scrapy/__init__.py

@@ -43,3 +43,8 @@ except ImportError:
     pass
 else:
     optional_features.add('django')
+
+from twisted import version as _txv
+twisted_version = (_txv.major, _txv.minor, _txv.micro)
+if twisted_version >= (11, 1, 0):
+    optional_features.add('http11')

scrapy/core/downloader/handlers/http.py

@@ -1,25 +1,11 @@
-"""Download handlers for http and https schemes
-"""
-from twisted.internet import reactor
-from scrapy.utils.misc import load_object
+from scrapy import optional_features
+from .http10 import HTTP10DownloadHandler
+
+if 'http11' in optional_features:
+    from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
+else:
+    HTTPDownloadHandler = HTTP10DownloadHandler
 
 
-class HttpDownloadHandler(object):
-
-    def __init__(self, settings):
-        self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
-        self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
-
-    def download_request(self, request, spider):
-        """Return a deferred for the HTTP download"""
-        factory = self.HTTPClientFactory(request)
-        self._connect(factory)
-        return factory.deferred
-
-    def _connect(self, factory):
-        host, port = factory.host, factory.port
-        if factory.scheme == 'https':
-            return reactor.connectSSL(host, port, factory,
-                                      self.ClientContextFactory())
-        else:
-            return reactor.connectTCP(host, port, factory)
+# backwards compatibility
+HttpDownloadHandler = HTTP10DownloadHandler

scrapy/core/downloader/handlers/http10.py

@@ -0,0 +1,25 @@
+"""Download handlers for http and https schemes
+"""
+from twisted.internet import reactor
+from scrapy.utils.misc import load_object
+
+
+class HTTP10DownloadHandler(object):
+
+    def __init__(self, settings):
+        self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
+        self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+
+    def download_request(self, request, spider):
+        """Return a deferred for the HTTP download"""
+        factory = self.HTTPClientFactory(request)
+        self._connect(factory)
+        return factory.deferred
+
+    def _connect(self, factory):
+        host, port = factory.host, factory.port
+        if factory.scheme == 'https':
+            return reactor.connectSSL(host, port, factory,
+                                      self.ClientContextFactory())
+        else:
+            return reactor.connectTCP(host, port, factory)

scrapy/core/downloader/handlers/http11.py

@@ -21,7 +21,7 @@ from scrapy import log
 
 
 
-class Http11DownloadHandler(object):
+class HTTP11DownloadHandler(object):
 
     def __init__(self, settings):
         self._pool = HTTPConnectionPool(reactor, persistent=True)

scrapy/settings/default_settings.py

@@ -53,8 +53,8 @@ DOWNLOAD_DELAY = 0
 DOWNLOAD_HANDLERS = {}
 DOWNLOAD_HANDLERS_BASE = {
     'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
-    'http': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler',
-    'https': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler',
+    'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
+    'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
     's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
 }
 

scrapy/tests/test_downloader_handlers.py

@@ -12,8 +12,9 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
 from w3lib.url import path_to_file_uri
 
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
-from scrapy.core.downloader.handlers.http import HttpDownloadHandler
-from scrapy.core.downloader.handlers.http11 import Http11DownloadHandler
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
+from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
+from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.spider import BaseSpider
 from scrapy.http import Request
@@ -48,7 +49,7 @@ class FileTestCase(unittest.TestCase):
 
 class HttpTestCase(unittest.TestCase):
 
-    download_handler_cls = HttpDownloadHandler
+    download_handler_cls = HTTPDownloadHandler
 
     def setUp(self):
         name = self.mktemp()
@@ -140,11 +141,20 @@ class HttpTestCase(unittest.TestCase):
         return d
 
 
+class DeprecatedHttpTestCase(HttpTestCase):
+    """HTTP 1.0 test case"""
+    download_handler_cls = HttpDownloadHandler
+
+
+class Http10TestCase(HttpTestCase):
+    """HTTP 1.0 test case"""
+    download_handler_cls = HTTP10DownloadHandler
+
+
 class Http11TestCase(HttpTestCase):
     """HTTP 1.1 test case"""
-    download_handler_cls = Http11DownloadHandler
-
-    if twisted.__version__.split('.') < (11, 1, 0):
+    download_handler_cls = HTTP11DownloadHandler
+    if 'http11' not in optional_features:
         skip = 'HTTP1.1 not supported in twisted < 11.1.0'
 
 
@@ -159,8 +169,7 @@ class UriResource(resource.Resource):
 
 
 class HttpProxyTestCase(unittest.TestCase):
-
-    download_handler_cls = HttpDownloadHandler
+    download_handler_cls = HTTPDownloadHandler
 
     def setUp(self):
         site = server.Site(UriResource(), timeout=None)
@@ -199,10 +208,18 @@ class HttpProxyTestCase(unittest.TestCase):
         return self.download_request(request, BaseSpider('foo')).addCallback(_test)
 
 
-class Http11ProxyTestCase(HttpProxyTestCase):
-    download_handler_cls = Http11DownloadHandler
+class DeprecatedHttpProxyTestCase(unittest.TestCase):
+    """Old deprecated reference to http10 downloader handler"""
+    download_handler_cls = HttpDownloadHandler
 
-    if twisted.__version__.split('.') < (11, 1, 0):
+
+class Http10ProxyTestCase(HttpProxyTestCase):
+    download_handler_cls = HTTP10DownloadHandler
+
+
+class Http11ProxyTestCase(HttpProxyTestCase):
+    download_handler_cls = HTTP11DownloadHandler
+    if 'http11' not in optional_features:
         skip = 'HTTP1.1 not supported in twisted < 11.1.0'
 
 

scrapy/xlib/tx/__init__.py

@@ -1,9 +1,8 @@
-import twisted
-txver = twisted.__version__.split('.')
-if txver > (13, 0, 0):
+from scrapy import twisted_version
+if twisted_version > (13, 0, 0):
     from twisted.web import client
     from twisted.internet import endpoints
-if txver >= (11, 1, 0):
+if twisted_version >= (11, 1, 0):
     from . import client, endpoints
 else:
     from scrapy.exceptions import NotSupported