refactor: move agents & context-factory

scrapy/core/downloader/contextfactory.py

@@ -1,10 +1,12 @@
 import warnings
 
 from OpenSSL import SSL
+from twisted.internet._sslverify import _setAcceptableProtocols
 from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
 from twisted.web.client import BrowserLikePolicyForHTTPS
 from twisted.web.iweb import IPolicyForHTTPS
 from zope.interface.declarations import implementer
+from zope.interface.verify import verifyObject
 
 from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
 from scrapy.utils.misc import create_instance, load_object
@@ -84,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
     The default OpenSSL method is ``TLS_METHOD`` (also called
     ``SSLv23_METHOD``) which allows TLS protocol negotiation.
     """
-    def creatorForNetloc(self, hostname, port):
 
+    def creatorForNetloc(self, hostname, port):
         # trustRoot set to platformTrust() will use the platform's root CAs.
         #
         # This means that a website like https://www.cacert.org will be rejected
@@ -97,6 +99,24 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
         )
 
 
+@implementer(IPolicyForHTTPS)
+class AcceptableProtocolsContextFactory:
+    """Context factory to used to override the acceptable protocols
+    to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN
+    negotiation.
+    """
+
+    def __init__(self, context_factory, acceptable_protocols):
+        verifyObject(IPolicyForHTTPS, context_factory)
+        self._wrapped_context_factory = context_factory
+        self._acceptable_protocols = acceptable_protocols
+
+    def creatorForNetloc(self, hostname, port):
+        options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
+        _setAcceptableProtocols(options._ctx, self._acceptable_protocols)
+        return options
+
+
 def load_context_factory_from_settings(settings, crawler):
     ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
     context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])

scrapy/core/downloader/handlers/http2.py

@@ -1,16 +1,15 @@
 import warnings
 from time import time
-from typing import Optional, Tuple
+from typing import Optional
 from urllib.parse import urldefrag
 
-from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
-from twisted.web.client import BrowserLikePolicyForHTTPS, URI
+from twisted.web.client import URI
 
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
-from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
+from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
@@ -41,30 +40,6 @@ class H2DownloadHandler:
         self._pool.close_connections()
 
 
-class ScrapyProxyH2Agent(H2Agent):
-    def __init__(
-        self, reactor: ReactorBase,
-        proxy_uri: URI, pool: H2ConnectionPool,
-        context_factory=BrowserLikePolicyForHTTPS(),
-        connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
-    ) -> None:
-        super(ScrapyProxyH2Agent, self).__init__(
-            reactor=reactor,
-            pool=pool,
-            context_factory=context_factory,
-            connect_timeout=connect_timeout,
-            bind_address=bind_address
-        )
-        self._proxy_uri = proxy_uri
-
-    def get_endpoint(self, uri: URI):
-        return self.endpoint_factory.endpointForURI(self._proxy_uri)
-
-    def get_key(self, uri: URI) -> Tuple:
-        """We use the proxy uri instead of uri obtained from request url"""
-        return "http-proxy", self._proxy_uri.host, self._proxy_uri.port
-
-
 class ScrapyH2Agent:
     _Agent = H2Agent
     _ProxyAgent = ScrapyProxyH2Agent

scrapy/core/http2/agent.py

@@ -2,17 +2,14 @@ from collections import deque
 from typing import Deque, Dict, List, Tuple, Optional
 
 from twisted.internet import defer
-from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions
 from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
 from twisted.internet.endpoints import HostnameEndpoint
 from twisted.python.failure import Failure
 from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
 from twisted.web.error import SchemeNotSupported
-from twisted.web.iweb import IPolicyForHTTPS
-from zope.interface import implementer
-from zope.interface.verify import verifyObject
 
+from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
 from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
 from scrapy.http.request import Request
 from scrapy.settings import Settings
@@ -96,19 +93,6 @@ class H2ConnectionPool:
             conn.transport.abortConnection()
 
 
-@implementer(IPolicyForHTTPS)
-class AcceptableProtocolsContextFactory:
-    def __init__(self, context_factory, acceptable_protocols: List[bytes]) -> None:
-        verifyObject(IPolicyForHTTPS, context_factory)
-        self._wrapped_context_factory = context_factory
-        self._acceptable_protocols = acceptable_protocols
-
-    def creatorForNetloc(self, hostname, port) -> ClientTLSOptions:
-        options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
-        _setAcceptableProtocols(options._ctx, self._acceptable_protocols)
-        return options
-
-
 class H2Agent:
     def __init__(
         self, reactor: ReactorBase, pool: H2ConnectionPool,
@@ -144,3 +128,27 @@ class H2Agent:
         d = self._pool.get_connection(key, uri, endpoint)
         d.addCallback(lambda conn: conn.request(request, spider))
         return d
+
+
+class ScrapyProxyH2Agent(H2Agent):
+    def __init__(
+        self, reactor: ReactorBase,
+        proxy_uri: URI, pool: H2ConnectionPool,
+        context_factory=BrowserLikePolicyForHTTPS(),
+        connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
+    ) -> None:
+        super(ScrapyProxyH2Agent, self).__init__(
+            reactor=reactor,
+            pool=pool,
+            context_factory=context_factory,
+            connect_timeout=connect_timeout,
+            bind_address=bind_address
+        )
+        self._proxy_uri = proxy_uri
+
+    def get_endpoint(self, uri: URI):
+        return self.endpoint_factory.endpointForURI(self._proxy_uri)
+
+    def get_key(self, uri: URI) -> Tuple:
+        """We use the proxy uri instead of uri obtained from request url"""
+        return "http-proxy", self._proxy_uri.host, self._proxy_uri.port