Make Twisted[http2] installation optional (#5113)

Co-authored-by: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>

conftest.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
 import pytest
+from twisted.web.http import H2_ENABLED
 
 from scrapy.utils.reactor import install_reactor
 
@@ -25,6 +26,14 @@ for line in open('tests/ignores.txt'):
     if file_path and file_path[0] != '#':
         collect_ignore.append(file_path)
 
+if not H2_ENABLED:
+    collect_ignore.extend(
+        (
+            'scrapy/core/downloader/handlers/http2.py',
+            *_py_files("scrapy/core/http2"),
+        )
+    )
+
 
 @pytest.fixture()
 def chdir(tmpdir):

docs/topics/settings.rst

@@ -680,12 +680,16 @@ handler (without replacement), place this in your ``settings.py``::
 
 .. _http2:
 
-The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
-:setting:`DOWNLOAD_HANDLERS` as follows::
+The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
 
-    DOWNLOAD_HANDLERS = {
-        'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
-    }
+#.  Install ``Twisted[http2]>=17.9.0`` to install the packages required to
+    enable HTTP/2 support in Twisted.
+
+#.  Update :setting:`DOWNLOAD_HANDLERS` as follows::
+
+        DOWNLOAD_HANDLERS = {
+            'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
+        }
 
 .. warning::
 

setup.py

@@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support():
 
 
 install_requires = [
-    'Twisted[http2]>=17.9.0',
+    'Twisted>=17.9.0',
     'cryptography>=2.0',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
@@ -31,7 +31,6 @@ install_requires = [
     'zope.interface>=4.1.3',
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
-    'h2>=3.0,<4.0',
     'setuptools',
 ]
 extras_require = {}

tests/test_downloader_handlers_http2.py

@@ -1,5 +1,5 @@
 import json
-from unittest import mock
+from unittest import mock, skipIf
 
 from pytest import mark
 from testfixtures import LogCapture
@@ -7,8 +7,8 @@ from twisted.internet import defer, error, reactor
 from twisted.trial import unittest
 from twisted.web import server
 from twisted.web.error import SchemeNotSupported
+from twisted.web.http import H2_ENABLED
 
-from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.misc import create_instance
@@ -21,11 +21,17 @@ from tests.test_downloader_handlers import (
 )
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2TestCase(Https11TestCase):
+
     scheme = 'https'
-    download_handler_cls = H2DownloadHandler
     HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
 
+    @classmethod
+    def setUpClass(cls):
+        from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+        cls.download_handler_cls = H2DownloadHandler
+
     def test_protocol(self):
         request = Request(self.getURL("host"), method="GET")
         d = self.download_request(request, Spider("foo"))
@@ -187,9 +193,14 @@ class Https2InvalidDNSPattern(Https2TestCase):
         super(Https2InvalidDNSPattern, self).setUp()
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2CustomCiphers(Https11CustomCiphers):
     scheme = 'https'
-    download_handler_cls = H2DownloadHandler
+
+    @classmethod
+    def setUpClass(cls):
+        from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+        cls.download_handler_cls = H2DownloadHandler
 
 
 class Http2MockServerTestCase(Http11MockServerTestCase):
@@ -201,6 +212,7 @@ class Http2MockServerTestCase(Http11MockServerTestCase):
     }
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2ProxyTestCase(Http11ProxyTestCase):
     # only used for HTTPS tests
     keyfile = 'keys/localhost.key'
@@ -209,9 +221,13 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
     scheme = 'https'
     host = u'127.0.0.1'
 
-    download_handler_cls = H2DownloadHandler
     expected_http_proxy_request_body = b'/'
 
+    @classmethod
+    def setUpClass(cls):
+        from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+        cls.download_handler_cls = H2DownloadHandler
+
     def setUp(self):
         site = server.Site(UriResource(), timeout=None)
         self.port = reactor.listenSSL(

tests/test_http2_client_protocol.py

@@ -5,10 +5,9 @@ import re
 import shutil
 import string
 from ipaddress import IPv4Address
-from unittest import mock
+from unittest import mock, skipIf
 from urllib.parse import urlencode
 
-from h2.exceptions import InvalidBodyLengthError
 from twisted.internet import reactor
 from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
@@ -17,12 +16,10 @@ from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certif
 from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
 from twisted.web.client import ResponseFailed, URI
-from twisted.web.http import Request as TxRequest
+from twisted.web.http import H2_ENABLED, Request as TxRequest
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.static import File
 
-from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
-from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
 from scrapy.http import Request, Response, JsonRequest
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
@@ -173,6 +170,7 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
     return PrivateCertificate.loadPEM(pem)
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2ClientProtocolTestCase(TestCase):
     scheme = 'https'
     key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key')
@@ -220,6 +218,7 @@ class Https2ClientProtocolTestCase(TestCase):
         uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8'))
 
         self.conn_closed_deferred = Deferred()
+        from scrapy.core.http2.protocol import H2ClientFactory
         h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
         client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
         self.client = yield client_endpoint.connect(h2_client_factory)
@@ -426,6 +425,7 @@ class Https2ClientProtocolTestCase(TestCase):
 
         def assert_failure(failure: Failure):
             self.assertTrue(len(failure.value.reasons) > 0)
+            from h2.exceptions import InvalidBodyLengthError
             self.assertTrue(any(
                 isinstance(error, InvalidBodyLengthError)
                 for error in failure.value.reasons
@@ -511,6 +511,7 @@ class Https2ClientProtocolTestCase(TestCase):
 
         def assert_inactive_stream(failure):
             self.assertIsNotNone(failure.check(ResponseFailed))
+            from scrapy.core.http2.stream import InactiveStreamClosed
             self.assertTrue(any(
                 isinstance(e, InactiveStreamClosed)
                 for e in failure.value.reasons
@@ -596,6 +597,7 @@ class Https2ClientProtocolTestCase(TestCase):
         request = Request(url)
 
         def assert_invalid_hostname(failure: Failure):
+            from scrapy.core.http2.stream import InvalidHostname
             self.assertIsNotNone(failure.check(InvalidHostname))
             error_msg = str(failure.value)
             self.assertIn('localhost', error_msg)
@@ -633,6 +635,7 @@ class Https2ClientProtocolTestCase(TestCase):
 
         def assert_timeout_error(failure: Failure):
             for err in failure.value.reasons:
+                from scrapy.core.http2.protocol import H2ClientProtocol
                 if isinstance(err, TimeoutError):
                     self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err))
                     break

tox.ini

@@ -50,6 +50,8 @@ commands =
 basepython = python3
 deps =
     {[testenv]deps}
+    # Twisted[http2] is required to import some files
+    Twisted[http2]>=17.9.0
     pytest-flake8
 commands =
     py.test --flake8 {posargs:docs scrapy tests}
@@ -57,12 +59,7 @@ commands =
 [testenv:pylint]
 basepython = python3
 deps =
-    {[testenv]deps}
-    # Optional dependencies
-    boto
-    reppy
-    robotexclusionrulesparser
-    # Test dependencies
+    {[testenv:extra-deps]deps}
     pylint
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
@@ -119,9 +116,11 @@ setenv =
 [testenv:extra-deps]
 deps =
     {[testenv]deps}
+    boto
     reppy
     robotexclusionrulesparser
     Pillow>=4.0.0
+    Twisted[http2]>=17.9.0
 
 [testenv:asyncio]
 commands =